** IT 活动日历

核心创意源自 @too 的一个想法，核心代码源自 @TechQuery 的《NodeJS 网页爬虫一小时实战》课程。

基本用法

活动列表

命令行

npm install @fcc-cdc/it-events -g

# 以间隔 0.5 秒的速度把所有活动下载到桌面的一个文件
it-events list -i 0.5 1> ~/Desktop/it-events.yml

Node.JS 模块

import updateEvents, { descendDate } from '@fcc-cdc/it-events';

(async () => {
    const list = [];

    for await (let item of updateEvents(list, 0.5)) list.push(item);

    console.log([...new Set(list)].sort(descendDate));
})();

议程列表

命令行

# 议程在首页
it-events agenda https://www.bagevent.com/event/8199059

# 议程在内页
it-events agenda https://www.bagevent.com/event/6840909/p/430761

Node.js 模块

import { BagEventAgenda } from 'it-events';

// 议程在首页
new BagEventAgenda().saveList('https://www.bagevent.com/event/8199059');

// 议程在内页
new BagEventAgenda().saveList(
    'https://www.bagevent.com/event/6840909/p/430761'
);

活动列表的 OOP 重构

目标模块

IT-events/source/crawler.ts

Lines 6 to 206 in c34afa8

 export async function* HuoDongXing(all?: boolean) { 

 for (let page = 1, empty = true; ; page++) { 

 const URL = 

 'https://www.huodongxing.com/eventlist?' + 

 new URLSearchParams({ 

 orderby: 'n', 

 status: all ? '' : '1', 

 tag: 'IT互联网', 

 city: '全部', 

 page: page + '' 

 }); 

 for await (const item of eventList( 

 URL, 

 '.search-tab-content-list .search-tab-content-item', 

 '.item-title', 

 '.item-data', 

 '.item-dress', 

 '.item-logo', 

 '.item-title' 

 )) { 

 empty = false; 

 const [start, end] = (item.start as string).split('-'); 

 yield { 

 ...item, 

 start: makeDate(start), 

 end: makeDate(end) 

 } as Event; 

 } 

 if (empty) break; 

 } 

 } 

 export async function* SegmentFault(all?: boolean) { 

 for (let page = 1, empty = true; ; page++) { 

 const URL = 'https://segmentfault.com/events?page=' + page, 

 now = new Date(); 

 for await (const item of eventList( 

 URL, 

 '.all-event-list .widget-event', 

 '.title', 

 '.widget-event__meta > :first-child', 

 '.widget-event__meta > :last-child', 

 '.widget-event__banner', 

 '.title > a' 

 )) { 

 empty = false; 

 const start = makeDate((item.start as string).slice(3)); 

 if (!all && now > start) return; 

 yield { 

 ...item, 

 start, 

 end: null, 

 address: (item.address as string).slice(3) 

 } as Event; 

 } 

 if (empty) break; 

 } 

 } 

 export async function* JueJin(all?: boolean) { 

 for (let page = 1; ; page++) { 

 const URI = 

 'https://event-storage-api-ms.juejin.im/v2/getEventList?' + 

 new URLSearchParams({ 

 src: 'web', 

 orderType: 'startTime', 

 pageNum: page + '' 

 }), 

 now = new Date(); 

 const { d: list } = await (await fetch(URI)).json(); 

 if (!list?.[0]) break; 

 console.warn(URI); 

 for (const { 

 title, 

 eventUrl, 

 tagInfo, 

 content, 

 startTime, 

 endTime, 

 city, 

 screenshot 

 } of list) { 

 const start = makeDate(startTime), 

 end = makeDate(endTime); 

 if (!all && now > start) return; 

 yield { 

 title, 

 start, 

 end, 

 address: city, 

 tags: tagInfo.map(({ title }) => title), 

 summary: content, 

 link: new URL(eventUrl), 

 banner: new URL(screenshot) 

 } as Event; 

 } 

 } 

 } 

 export async function* BaiGe(all?: boolean) { 

 const { 

 window: { 

 document: { head } 

 } 

 } = await JSDOM.fromURL( 

 'https://www.bagevent.com/eventlist.html?f=1&tag=17&r=orderByNew' 

 ); 

 const { paramMap, imgDomain, mainDomain } = new Function(`${ 

  [ 

  ...head.querySelectorAll<HTMLScriptElement>('script:not(:empty)') 

  ].find(code => /var param = \{[\s\S]+\}/.test(code.text)).text 

  } 

  return param;`)(); 

 for (let page = 1; ; page++) { 

 paramMap.pagingPage = page; 

 const URI = `${mainDomain}/load/loadSearchEventList.do?${new URLSearchParams( 

  paramMap 

  )}`, 

 now = new Date(); 

 const { list } = ( 

 await (await fetch(URI)).json() 

 ).resultObject.valueList; 

 if (!list?.[0]) break; 

 console.warn(URI); 

 for (const { 

 event_name, 

 start_time, 

 address, 

 logo, 

 event_id 

 } of list) { 

 const start = makeDate(start_time); 

 if (!all && now > start) return; 

 yield { 

 title: event_name, 

 start, 

 address, 

 banner: imgDomain + logo, 

 link: new URL(mainDomain + '/event/' + event_id) 

 } as Event; 

 } 

 } 

 } 

 export async function* OSChina(all?: boolean) { 

 for (let page = 1, empty = true; ; page++) { 

 const body = new URLSearchParams({ 

 tab: 'latest', 

 time: 'all', 

 p: page + '' 

 }), 

 URL = 'https://www.oschina.net/action/ajax/get_more_event_list', 

 now = new Date(); 

 const data = await (await fetch(URL, { method: 'POST', body })).text(); 

 for await (const item of eventList( 

 new JSDOM(data, { url: URL + '?' + body }), 

 '.event-item', 

 '.summary', 

 '.when-where > label:first-of-type', 

 '.when-where > label:last-of-type', 

 '.item-banner img', 

 '.item-banner > a' 

 )) { 

 empty = false; 

 const start = makeDate(item.start as string); 

 if (!all && now > start) return; 

 yield { ...item, start, end: null } as Event; 

 } 

 if (empty) break; 

 } 

 }

继承基类

IT-events/source/core/Crawler.ts

Lines 11 to 39 in c34afa8

 export abstract class DataCrawler<T> { 

 declare ['constructor']: typeof DataCrawler; 

 static baseURI = ''; 

 static schema: URLPattern; 

 abstract getList(URI: string): AsyncGenerator<T>; 

 abstract getItem(URI: string): Promise<T>; 

 makeYAML(list: T[]) { 

 return stringify(list); 

 } 

 @logTime 

 async saveList(URI: string) { 

 const list: T[] = [], 

 folder = join(process.cwd(), 'temp', new URL(URI).pathname); 

 for await (const item of this.getList(URI)) { 

 console.log(item); 

 list.push(item); 

 } 

 const file = await saveFile(this.makeYAML(list), folder, 'index.yml'); 

 return { list, folder, file }; 

 } 

 }

参考实现

IT-events/source/Agenda/core.ts

Lines 22 to 83 in c34afa8

 export abstract class AgendaCrawler< 

 A extends Agenda = Agenda, 

 M extends Mentor = Mentor, 

 F extends Forum = Forum 

 > extends DataCrawler<A> { 

 mentors: M[] = []; 

 forums: F[] = []; 

 override makeYAML(agendas: A[]) { 

 return stringify({ 

 mentors: this.mentors, 

 forums: this.forums, 

 agendas 

 }); 

 } 

 makeCSV(agendas: A[]) { 

 const { mentors, forums } = this; 

 return { 

 mentors, 

 forums, 

 agendas: agendas.map(({ mentor, forum, ...agenda }) => ({ 

 ...agenda, 

 mentor: mentor?.name, 

 forum: forum?.name 

 })) 

 }; 

 } 

 @logTime 

 async saveCSV(agendas: A[], folder: string) { 

 for (const [name, rows] of Object.entries(this.makeCSV(agendas))) 

 await saveFile(stringifyCSV(rows), folder, `${name}.csv`); 

 } 

 @logTime 

 async saveImagesTo(folder: string) { 

 for (const { avatar } of this.mentors) 

 if (avatar) { 

 const response = await fetch(avatar); 

 const buffer = Buffer.from(await response.arrayBuffer()); 

 const { ext } = await fromBuffer(buffer); 

 await saveFile( 

 buffer, 

 folder, 

 'image', 

 `${parse(avatar).name}.${ext}` 

 ); 

 } 

 } 

 override async saveList(URI: string) { 

 const { list, folder, file } = await super.saveList(URI); 

 await this.saveCSV(list, folder); 

 await this.saveImagesTo(folder); 

 return { list, folder, file }; 

 } 

 }

	export async function* HuoDongXing(all?: boolean) {
	for (let page = 1, empty = true; ; page++) {
	const URL =
	'https://www.huodongxing.com/eventlist?' +
	new URLSearchParams({
	orderby: 'n',
	status: all ? '' : '1',
	tag: 'IT互联网',
	city: '全部',
	page: page + ''
	});

	for await (const item of eventList(
	URL,
	'.search-tab-content-list .search-tab-content-item',
	'.item-title',
	'.item-data',
	'.item-dress',
	'.item-logo',
	'.item-title'
	)) {
	empty = false;

	const [start, end] = (item.start as string).split('-');

	yield {
	...item,
	start: makeDate(start),
	end: makeDate(end)
	} as Event;
	}

	if (empty) break;
	}
	}

	export async function* SegmentFault(all?: boolean) {
	for (let page = 1, empty = true; ; page++) {
	const URL = 'https://segmentfault.com/events?page=' + page,
	now = new Date();

	for await (const item of eventList(
	URL,
	'.all-event-list .widget-event',
	'.title',
	'.widget-event__meta > :first-child',
	'.widget-event__meta > :last-child',
	'.widget-event__banner',
	'.title > a'
	)) {
	empty = false;

	const start = makeDate((item.start as string).slice(3));

	if (!all && now > start) return;

	yield {
	...item,
	start,
	end: null,
	address: (item.address as string).slice(3)
	} as Event;
	}

	if (empty) break;
	}
	}

	export async function* JueJin(all?: boolean) {
	for (let page = 1; ; page++) {
	const URI =
	'https://event-storage-api-ms.juejin.im/v2/getEventList?' +
	new URLSearchParams({
	src: 'web',
	orderType: 'startTime',
	pageNum: page + ''
	}),
	now = new Date();

	const { d: list } = await (await fetch(URI)).json();

	if (!list?.[0]) break;

	console.warn(URI);

	for (const {
	title,
	eventUrl,
	tagInfo,
	content,
	startTime,
	endTime,
	city,
	screenshot
	} of list) {
	const start = makeDate(startTime),
	end = makeDate(endTime);

	if (!all && now > start) return;

	yield {
	title,
	start,
	end,
	address: city,
	tags: tagInfo.map(({ title }) => title),
	summary: content,
	link: new URL(eventUrl),
	banner: new URL(screenshot)
	} as Event;
	}
	}
	}

	export async function* BaiGe(all?: boolean) {
	const {
	window: {
	document: { head }
	}
	} = await JSDOM.fromURL(
	'https://www.bagevent.com/eventlist.html?f=1&tag=17&r=orderByNew'
	);

	const { paramMap, imgDomain, mainDomain } = new Function(`${
	[
	...head.querySelectorAll<HTMLScriptElement>('script:not(:empty)')
	].find(code => /var param = \{[\s\S]+\}/.test(code.text)).text
	}
	return param;`)();

	for (let page = 1; ; page++) {
	paramMap.pagingPage = page;

	const URI = `${mainDomain}/load/loadSearchEventList.do?${new URLSearchParams(
	paramMap
	)}`,
	now = new Date();

	const { list } = (
	await (await fetch(URI)).json()
	).resultObject.valueList;

	if (!list?.[0]) break;

	console.warn(URI);

	for (const {
	event_name,
	start_time,
	address,
	logo,
	event_id
	} of list) {
	const start = makeDate(start_time);

	if (!all && now > start) return;

	yield {
	title: event_name,
	start,
	address,
	banner: imgDomain + logo,
	link: new URL(mainDomain + '/event/' + event_id)
	} as Event;
	}
	}
	}

	export async function* OSChina(all?: boolean) {
	for (let page = 1, empty = true; ; page++) {
	const body = new URLSearchParams({
	tab: 'latest',
	time: 'all',
	p: page + ''
	}),
	URL = 'https://www.oschina.net/action/ajax/get_more_event_list',
	now = new Date();

	const data = await (await fetch(URL, { method: 'POST', body })).text();

	for await (const item of eventList(
	new JSDOM(data, { url: URL + '?' + body }),
	'.event-item',
	'.summary',
	'.when-where > label:first-of-type',
	'.when-where > label:last-of-type',
	'.item-banner img',
	'.item-banner > a'
	)) {
	empty = false;

	const start = makeDate(item.start as string);

	if (!all && now > start) return;

	yield { ...item, start, end: null } as Event;
	}

	if (empty) break;
	}
	}

	export abstract class DataCrawler<T> {
	declare ['constructor']: typeof DataCrawler;

	static baseURI = '';
	static schema: URLPattern;

	abstract getList(URI: string): AsyncGenerator<T>;

	abstract getItem(URI: string): Promise<T>;

	makeYAML(list: T[]) {
	return stringify(list);
	}

	@logTime
	async saveList(URI: string) {
	const list: T[] = [],
	folder = join(process.cwd(), 'temp', new URL(URI).pathname);

	for await (const item of this.getList(URI)) {
	console.log(item);

	list.push(item);
	}
	const file = await saveFile(this.makeYAML(list), folder, 'index.yml');

	return { list, folder, file };
	}
	}

	export abstract class AgendaCrawler<
	A extends Agenda = Agenda,
	M extends Mentor = Mentor,
	F extends Forum = Forum
	> extends DataCrawler<A> {
	mentors: M[] = [];
	forums: F[] = [];

	override makeYAML(agendas: A[]) {
	return stringify({
	mentors: this.mentors,
	forums: this.forums,
	agendas
	});
	}

	makeCSV(agendas: A[]) {
	const { mentors, forums } = this;

	return {
	mentors,
	forums,
	agendas: agendas.map(({ mentor, forum, ...agenda }) => ({
	...agenda,
	mentor: mentor?.name,
	forum: forum?.name
	}))
	};
	}

	@logTime
	async saveCSV(agendas: A[], folder: string) {
	for (const [name, rows] of Object.entries(this.makeCSV(agendas)))
	await saveFile(stringifyCSV(rows), folder, `${name}.csv`);
	}

	@logTime
	async saveImagesTo(folder: string) {
	for (const { avatar } of this.mentors)
	if (avatar) {
	const response = await fetch(avatar);
	const buffer = Buffer.from(await response.arrayBuffer());
	const { ext } = await fromBuffer(buffer);

	await saveFile(
	buffer,
	folder,
	'image',
	`${parse(avatar).name}.${ext}`
	);
	}
	}

	override async saveList(URI: string) {
	const { list, folder, file } = await super.saveList(URI);

	await this.saveCSV(list, folder);
	await this.saveImagesTo(folder);

	return { list, folder, file };
	}
	}

freecodecamp-chengdu / it-events Goto Github PK

it-events's Introduction

** IT 活动日历

基本用法

活动列表

命令行

Node.JS 模块

议程列表

命令行

Node.js 模块

it-events's People

Contributors

Stargazers

Watchers

Forkers

it-events's Issues

目标模块

继承基类

参考实现

Recommend Projects

Recommend Topics

Recommend Org