node-streams's Introduction

node-streams

How to run this project

type npm install on your terminal inside the project
run command node index.js

Issues?

why code is writing a single response from the API in the excel file when I'm iterating till 20 pages of the API
Other warning is

(node:9561) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 close listeners added to [Stream]. Use emitter.setMaxListeners() to increase limit

node-streams's People

Contributors

Watchers

node-streams's Issues

My solution

import fetch from 'node-fetch';
import {
  Readable,
  pipeline,
  Writable
} from 'stream';
import exceljs from 'exceljs';
import util from 'util';

const URL = 'https://jsonplaceholder.typicode.com/posts/'

const cpipeline = util.promisify(pipeline);

class ExcelStream extends Writable {
  constructor(options = {}) {
    super({
      ...options,
      objectMode: true
    });
    this.i = 0;
    const o = {
      filename: "./test-file.xlsx",
      useStyles: true,
      useSharedStrings: true,
    };
    this.workbook = new exceljs.stream.xlsx.WorkbookWriter(o);
    this.workbook.creator = "ABCDEF";
    this.workbook.created = new Date();
    this.sheet = this.workbook.addWorksheet("Discussion Event");
    this.sheet.columns = [{
      header: "testColumn",
      width: 10,
      style: {
        font: {
          bold: true,
        },
      },
    }, ];
  }

  _write (chunk, encoding, done) {
    console.log(chunk)
    try {
      this.i += 1;
      this.sheet.addRow([chunk]).commit();
    } catch (error) {
      console.log("Error while transforming data - commit failed");
      done(error)
      return
    }
    done()
  }

  _flush(cb) {
    try {
      this.sheet.commit();
      this.workbook.commit().then(() => cb(), cb);
    } catch (error) {
      console.log("Error cannot commit workbook");
    }
  }
}

async function run() {
  let i = 1;
  await cpipeline(async function * () {
    while (true) {
      let u = URL + i
      const response = await fetch(u)
      const result = await response.json()
      if (result) {
        yield result
      } else {
        break
      }

      i++
      if (i == 20) break
    }
  }, new ExcelStream())
}

run()

Recommend Projects