vulcanize / eth-watcher-ts Goto Github PK

View Code? Open in Web Editor NEW

5.0 5.0 3.0 1.86 MB

Shell 0.16% TypeScript 94.42% JavaScript 1.28% PLpgSQL 0.10% Dockerfile 0.14% Makefile 0.04% HTML 1.18% EJS 2.68%

eth-watcher-ts's Introduction

vulcanize

eth-watcher-ts's People

Contributors

Stargazers

Watchers

Forkers

daodesigner ashwinphatak isabella232

eth-watcher-ts's Issues

Fix state/storage exception (FPS.sol)

State id: 276162362
State id: 276162363
payload StorageRequest(address,string,string)
hash 0x98161ddebfa4d4fd1c08afb4633d60e0fa659fe1e2c2aebef15f6f1b18df7be9
address dfea08d7c2b43498bfe32778334c9279956057f0
98161ddebfa4d4fd1c08afb4633d60e0fa659fe1e2c2aebef15f6f1b18df7be9
array [
  {
    name: 'uploader',
    value: '29b1d432a40f40f5418da2d4abf740e5e491629b',
    type: 'address'
  },
  {
    name: 'cid',
    value: 'QmTJYDuVWNnRn6g4AjRE5adZ9GwTm7Sgu2Zj75a3xibDRJ',
    type: 'string'
  },
  {
    name: 'config',
    value: '{hot:{enabled:true,allowUnfreeze:true,ipfs:{addTimeout:900},unfreezeMaxPrice:0},cold:{enabled:true,filecoin:{replicationFactor:1,dealMinDuration:518400,excludedMinersList:[],trustedMinersList:[],countryCodesList:[],renew:{enabled:true,threshold:1},address:adr1,maxPrice:100000000000,fastRetrieval:true,dealStartOffset:8640,verifiedDeal:true}},repairable:false}',
    type: 'string'
  }
]
State id: 276162364
State id: 276162365
State id: 276162366
State id: 276162367
State id: 276162368
State id: 276162369
State id: 276162370
query: START TRANSACTION
query: INSERT INTO "eth"."block"("key", "data") VALUES ($1, $2) ON CONFLICT  ( key )  DO UPDATE SET key = EXCLUDED.key, data = EXCLUDED.data RETURNING * -- PARAMETERS: ["/blocks/DMQI76YA2OL5YFXP6Q5S5WIISL4GHNKS5BBKQVGGR2NMARZNWQPELKY","\\xf9025ca0a82f02604007a6680f048726ad7fcf64c4cb38e44b6284c14c0f60b2685a93fca01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347940000000000000000000000000000000000000000a050195429536af0e2c6bd7f92136642742536a26f306bc0810c34defac07485f3a03a5cdb0bfeeaba8ac0c0431551071d5b3042aac9e7fe3c8852f04292215ca017a07f59c220b654b6a17304b3a289123bdbe99dc16af5b7b707159a4dae109b41fab901000020010001000000000040214000000000000000000480500000000000008000000400000264080020004040100000100000400002000000002000002020200000000400000000400208a00800800000000000080402040000000040000000000800088022204000402000100000080000201000000000002000001101000000200004008004000002005040010000000080001080000002000c008000420000020000000000000000000001000000000400000000a02000004c000018004020000040060000044022000030000108040000800040080000000020000000200000100000000008000000201000000000000000002002000000000800000080000183827dc48398b193833481828460971151b861d883010a04846765746888676f312e31362e33856c696e75780000000000000022346b1f5ae0498c4bcf4377fa9b1ea52258c12b48bec3690765afffb6020a3769b5a23fa53ba6b114d76c519ea9375d661f6cba98b27010a37714ed08ba713a00a00000000000000000000000000000000000000000000000000000000000000000880000000000000000"]
query: INSERT INTO "eth"."header_cids"("block_number", "block_hash", "parent_hash", "cid", "mh_key", "td", "node_id", "reward", "state_root", "tx_root", "receipt_root", "uncle_root", "bloom", "timestamp", "times_validated") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT  ( block_number, block_hash )  DO UPDATE SET parent_hash = EXCLUDED.parent_hash, cid = EXCLUDED.cid, mh_key = EXCLUDED.mh_key, td = EXCLUDED.td, node_id = EXCLUDED.node_id RETURNING * -- PARAMETERS: ["8551876","0x8ffb00d397dc16eff43b2ed90892f863b552e842a854c68e9ac0472db41e45ab","0xa82f02604007a6680f048726ad7fcf64c4cb38e44b6284c14c0f60b2685a93fc","bagiacgzar75qbu4x3qlo75b3f3mqrexymo2vf2ccvbkmndu2ybds3na6iwvq","/blocks/DMQI76YA2OL5YFXP6Q5S5WIISL4GHNKS5BBKQVGGR2NMARZNWQPELKY","14707028",1,"2062958958000000000","0x50195429536af0e2c6bd7f92136642742536a26f306bc0810c34defac07485f3","0x3a5cdb0bfeeaba8ac0c0431551071d5b3042aac9e7fe3c8852f04292215ca017","0x7f59c220b654b6a17304b3a289123bdbe99dc16af5b7b707159a4dae109b41fa","0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","\\x0020010001000000000040214000000000000000000480500000000000008000000400000264080020004040100000100000400002000000002000002020200000000400000000400208a00800800000000000080402040000000040000000000800088022204000402000100000080000201000000000002000001101000000200004008004000002005040010000000080001080000002000c008000420000020000000000000000000001000000000400000000a02000004c00001800402000004006000004402200003000010804000080004008000000002000000020000010000000000800000020100000000000000000200200000000080000008000","1620513105",1]
query: COMMIT
query: START TRANSACTION
query: INSERT INTO "eth"."block"("key", "data") VALUES ($1, $2) ON CONFLICT  ( key )  DO UPDATE SET key = EXCLUDED.key, data = EXCLUDED.data RETURNING * -- PARAMETERS: ["/blocks/DMQJZAYQN63FVQPNFKBS5I6ILNF5ZOXRKJDOCA7X4MH4YWFOUYN4F7I","\\xf902aa15843b9aca0083011ba194dfea08d7c2b43498bfe32778334c9279956057f080b90244f641090c000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002e516d544a59447556574e6e526e366734416a52453561645a394777546d37536775325a6a3735613378696244524a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001687b686f743a7b656e61626c65643a747275652c616c6c6f77556e667265657a653a747275652c697066733a7b61646454696d656f75743a3930307d2c756e667265657a654d617850726963653a307d2c636f6c643a7b656e61626c65643a747275652c66696c65636f696e3a7b7265706c69636174696f6e466163746f723a312c6465616c4d696e4475726174696f6e3a3531383430302c6578636c756465644d696e6572734c6973743a5b5d2c747275737465644d696e6572734c6973743a5b5d2c636f756e747279436f6465734c6973743a5b5d2c72656e65773a7b656e61626c65643a747275652c7468726573686f6c643a317d2c616464726573733a616472312c6d617850726963653a3130303030303030303030302c6661737452657472696576616c3a747275652c6465616c53746172744f66667365743a383634302c76657269666965644465616c3a747275657d7d2c72657061697261626c653a66616c73657d0000000000000000000000000000000000000000000000002ba0a1c847ed6eb2448ba247341848051491cf026055eaf23fc9fe41f4cfd4f12650a040193c061ff16fcf9baa6e8578d6128f5ceb05e83284d00998a6b9b02520d282"]
query: INSERT INTO "eth"."transaction_cids"("header_id", "tx_hash", "index", "cid", "mh_key", "dst", "src", "tx_data") VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT  ( header_id, tx_hash )  DO UPDATE SET index = EXCLUDED.index, cid = EXCLUDED.cid, mh_key = EXCLUDED.mh_key, dst = EXCLUDED.dst, src = EXCLUDED.src, tx_data = EXCLUDED.tx_data RETURNING * -- PARAMETERS: [7,"0x9c83106fb65ac1ed2a832ea3c85b4bdcbaf15246e103f7e30fcc58aea61bc2fd",25,"bagjqcgzatsbra35wlla62kudf2r4qw2l3s5pcusg4eb7pyypzrmk5jq3yl6q","/blocks/DMQJZAYQN63FVQPNFKBS5I6ILNF5ZOXRKJDOCA7X4MH4YWFOUYN4F7I","0xdFEa08D7c2B43498Bfe32778334c9279956057F0","0x29b1d432a40f40F5418DA2d4ABf740e5E491629B","\\xf641090c000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002e516d544a59447556574e6e526e366734416a52453561645a394777546d37536775325a6a3735613378696244524a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001687b686f743a7b656e61626c65643a747275652c616c6c6f77556e667265657a653a747275652c697066733a7b61646454696d656f75743a3930307d2c756e667265657a654d617850726963653a307d2c636f6c643a7b656e61626c65643a747275652c66696c65636f696e3a7b7265706c69636174696f6e466163746f723a312c6465616c4d696e4475726174696f6e3a3531383430302c6578636c756465644d696e6572734c6973743a5b5d2c747275737465644d696e6572734c6973743a5b5d2c636f756e747279436f6465734c6973743a5b5d2c72656e65773a7b656e61626c65643a747275652c7468726573686f6c643a317d2c616464726573733a616472312c6d617850726963653a3130303030303030303030302c6661737452657472696576616c3a747275652c6465616c53746172744f66667365743a383634302c76657269666965644465616c3a747275657d7d2c72657061697261626c653a66616c73657d000000000000000000000000000000000000000000000000"]
query: COMMIT
query: START TRANSACTION
INSERT INTO data.contract_id_1_event_id_1
		(event_id,contract_id,mh_key,header_id,data_uploader,data_cid,data_config)
		VALUES
		('1','1','/blocks/DMQLF5JKFPE6JBRUOD4SE4FKTV2Q2RHNSXIVZK77VNYFQ3GLZ4H7IBI','7','29b1d432a40f40f5418da2d4abf740e5e491629b','QmTJYDuVWNnRn6g4AjRE5adZ9GwTm7Sgu2Zj75a3xibDRJ','{hot:{enabled:true,allowUnfreeze:true,ipfs:{addTimeout:900},unfreezeMaxPrice:0},cold:{enabled:true,filecoin:{replicationFactor:1,dealMinDuration:518400,excludedMinersList:[],trustedMinersList:[],countryCodesList:[],renew:{enabled:true,threshold:1},address:adr1,maxPrice:100000000000,fastRetrieval:true,dealStartOffset:8640,verifiedDeal:true}},repairable:false}') RETURNING id;
query: INSERT INTO data.contract_id_1_event_id_1
		(event_id,contract_id,mh_key,header_id,data_uploader,data_cid,data_config)
		VALUES
		('1','1','/blocks/DMQLF5JKFPE6JBRUOD4SE4FKTV2Q2RHNSXIVZK77VNYFQ3GLZ4H7IBI','7','29b1d432a40f40f5418da2d4abf740e5e491629b','QmTJYDuVWNnRn6g4AjRE5adZ9GwTm7Sgu2Zj75a3xibDRJ','{hot:{enabled:true,allowUnfreeze:true,ipfs:{addTimeout:900},unfreezeMaxPrice:0},cold:{enabled:true,filecoin:{replicationFactor:1,dealMinDuration:518400,excludedMinersList:[],trustedMinersList:[],countryCodesList:[],renew:{enabled:true,threshold:1},address:adr1,maxPrice:100000000000,fastRetrieval:true,dealStartOffset:8640,verifiedDeal:true}},repairable:false}') RETURNING id;
query: COMMIT
query: SELECT "Address"."address_id" AS "Address_address_id", "Address"."address" AS "Address_address", "Address"."hash" AS "Address_hash" FROM "data"."addresses" "Address" WHERE "Address"."address" = $1 LIMIT 1 -- PARAMETERS: ["0x29b1d432a40f40f5418da2d4abf740e5e491629b"]
query: SELECT * FROM data.contract_id_1_address_slot_id_1 WHERE address_id=2;
query: UPDATE data.contract_id_1_state_id_1 a
					SET address_id=b.address_id
					FROM data.contract_id_1_address_slot_id_1 b
					WHERE
					  a.address_id IS NULL
					  AND a.uploaderToContent=b.hash
query failed: UPDATE data.contract_id_1_state_id_1 a
					SET address_id=b.address_id
					FROM data.contract_id_1_address_slot_id_1 b
					WHERE
					  a.address_id IS NULL
					  AND a.uploaderToContent=b.hash
error: error: column a.uploadertocontent does not exist
    at Parser.parseErrorMessage (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/parser.ts:357:11)
    at Parser.handlePacket (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/parser.ts:186:21)
    at Parser.parse (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/parser.ts:101:30)
    at Socket.<anonymous> (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/index.ts:7:48)
    at Socket.emit (node:events:369:20)
    at Socket.emit (node:domain:470:12)
    at addChunk (node:internal/streams/readable:313:12)
    at readableAddChunk (node:internal/streams/readable:288:9)
    at Socket.Readable.push (node:internal/streams/readable:227:10)
    at TCP.onStreamRead (node:internal/stream_base_commons:190:23) {
  length: 184,
  severity: 'ERROR',
  code: '42703',
  detail: undefined,
  hint: 'Perhaps you meant to reference the column "a.uploaderToContent".',
  position: '172',
  internalPosition: undefined,
  internalQuery: undefined,
  where: undefined,
  schema: undefined,
  table: undefined,
  column: undefined,
  dataType: undefined,
  constraint: undefined,
  file: 'parse_relation.c',
  line: '3349',
  routine: 'errorMissingColumn'
}
Unhandled Rejection at: Promise {
  <rejected> QueryFailedError: column a.uploadertocontent does not exist
      at new QueryFailedError (/home/nandit/Desktop/eth-watcher-ts/src/error/QueryFailedError.ts:9:9)
      at Query.callback (/home/nandit/Desktop/eth-watcher-ts/src/driver/postgres/PostgresQueryRunner.ts:195:30)
      at Query.handleError (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/query.js:128:19)
      at Client._handleErrorMessage (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/client.js:326:17)
      at Connection.emit (node:events:369:20)
      at Connection.emit (node:domain:470:12)
      at /home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/connection.js:109:12
      at Parser.parse (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/parser.ts:102:9)
      at Socket.<anonymous> (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/index.ts:7:48)
      at Socket.emit (node:events:369:20) {
    length: 184,
    severity: 'ERROR',
    code: '42703',
    detail: undefined,
    hint: 'Perhaps you meant to reference the column "a.uploaderToContent".',
    position: '172',
    internalPosition: undefined,
    internalQuery: undefined,
    where: undefined,
    schema: undefined,
    table: undefined,
    column: undefined,
    dataType: undefined,
    constraint: undefined,
    file: 'parse_relation.c',
    line: '3349',
    routine: 'errorMissingColumn',
    query: 'UPDATE data.contract_id_1_state_id_1 a\n' +
      '\t\t\t\t\tSET address_id=b.address_id\n' +
      '\t\t\t\t\tFROM data.contract_id_1_address_slot_id_1 b\n' +
      '\t\t\t\t\tWHERE\n' +
      '\t\t\t\t\t  a.address_id IS NULL\n' +
      '\t\t\t\t\t  AND a.uploaderToContent=b.hash',
    parameters: []
  }
} reason: QueryFailedError: column a.uploadertocontent does not exist
    at new QueryFailedError (/home/nandit/Desktop/eth-watcher-ts/src/error/QueryFailedError.ts:9:9)
    at Query.callback (/home/nandit/Desktop/eth-watcher-ts/src/driver/postgres/PostgresQueryRunner.ts:195:30)
    at Query.handleError (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/query.js:128:19)
    at Client._handleErrorMessage (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/client.js:326:17)
    at Connection.emit (node:events:369:20)
    at Connection.emit (node:domain:470:12)
    at /home/nandit/Desktop/eth-watcher-ts/node_modules/pg/lib/connection.js:109:12
    at Parser.parse (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/parser.ts:102:9)
    at Socket.<anonymous> (/home/nandit/Desktop/eth-watcher-ts/node_modules/pg-protocol/src/index.ts:7:48)
    at Socket.emit (node:events:369:20) {
  length: 184,
  severity: 'ERROR',
  code: '42703',
  detail: undefined,
  hint: 'Perhaps you meant to reference the column "a.uploaderToContent".',
  position: '172',
  internalPosition: undefined,
  internalQuery: undefined,
  where: undefined,
  schema: undefined,
  table: undefined,
  column: undefined,
  dataType: undefined,
  constraint: undefined,
  file: 'parse_relation.c',
  line: '3349',
  routine: 'errorMissingColumn',
  query: 'UPDATE data.contract_id_1_state_id_1 a\n' +
    '\t\t\t\t\tSET address_id=b.address_id\n' +
    '\t\t\t\t\tFROM data.contract_id_1_address_slot_id_1 b\n' +
    '\t\t\t\t\tWHERE\n' +
    '\t\t\t\t\t  a.address_id IS NULL\n' +
    '\t\t\t\t\t  AND a.uploaderToContent=b.hash',
  parameters: []
}
State id: 276162371
State id: 276162372
State id: 276162373

Database and graphql connection configuration

We need to define a database and graphql config in toml file

Structure:

[database]
    name     = "vulcanize"
    hostname = "localhost"
    port     = 5432
    user     = "postgres"
    password = ""

[graphql]
    uri = "http://localhost:5000"

Websocket Listener

Docker & CI

Build docker image
Docker compose
continuous-integration in Github Actions

Read Contracts settings from Database and store in Memory Storage

Deploy event watcher for FPS.sol (rinkeby)

Link to instructions https://hackmd.io/bXheQ2ItRumIlWSBzahoIw
Link to FPS.sol https://github.com/nandit123/eth-watcher-runner/blob/master/contracts/FPS.sol

Expose Postgraphile interface

Call eth_call method

Inject Postgraphile

Setup Dev Environment

Store to db

Graphql Listener

Skeleton app

Database migration tool
Controller, Service, Repository layer
Unit test tool

Database migration tool

Add Env Variable for each toml parameters

Expose Graphql interface on dapptools

Ineffective mark-compacts near heap limit Allocation failed

This is a follow-up issue for a problem that was debugged in slack by @ramilexe

add-contract.ts command also runs backfiller using childProcess.spawn and it doesn't inherit env from parent process.

Nodejs version is v15.14.0
Attempted fix NODE_OPTIONS=--max_old_space_size=8192

npx ts-node src/cli/add-contract.ts --name FPS --address ... --source-path ~/contract-tests/contracts/FPS.sol --artifact-path ~/contract-tests/artifacts/contracts/FPS.sol/FPS.json --starting-block 8498738

<--- Last few GCs --->

[60474:0x5c90660]  1582159 ms: Mark-sweep 3764.0 (4127.7) -> 3745.7 (4123.5) MB, 4864.7 / 0.0 ms  (average mu = 0.315, current mu = 0.285) task scavenge might not succeed
[60474:0x5c90660]  1582951 ms: Scavenge 3763.4 (4125.3) -> 3748.4 (4123.9) MB, 17.4 / 0.0 ms  (average mu = 0.315, current mu = 0.285) task 
[60474:0x5c90660]  1583353 ms: Scavenge 3764.7 (4126.0) -> 3750.8 (4125.3) MB, 18.5 / 0.0 ms  (average mu = 0.315, current mu = 0.285) task 

<--- JS stacktrace --->

FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory

stderr:  1: 0xa89e60 node::Abort() [node]

stderr:  2: 0x9ade29 node::FatalError(char const*, char const*) [node]

stderr:  3: 0xc7583e v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [node]

stderr:  4: 0xc75bb7 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [node]
 5: 0xe3f6d5  [node]
 6: 0xe4027c  [node]
 7: 0xe4dc0b v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [node]
 8: 0xe5190c v8::internal::Heap::AllocateRawWithRetryOrFailSlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [node]
 9: 0xe157da v8::internal::Factory::NewFillerObject(int, bool, v8::internal::AllocationType, v8::internal::AllocationOrigin) [node]
10: 0x116d4ab v8::internal::Runtime_AllocateInYoungGeneration(int, unsigned long*, v8::internal::Isolate*) [node]
11: 0x15045f9  [node]

Unhandled Rejection at: Promise { <rejected> null } reason: null

CLI command to add new contract to watcher.

solc flag to get storage layout in compiled artifact
insert data into events, states table, get IDs
npm run dev
New tables created by watcher
query contract variable from new table (name contains contract and variable names) over postgraphile

Database Design

Convert IPLD to internal data format

Filter incoming data by using Contract Settings from db

Create dynamic tables

On each contract event we need to create separate table in data schema:

    event MessageChanged(string message, byte a, int b, address c, byte32 d);

    // MessageChanged("abc", "t", 100, "0xabcasdasdasd", "abcd");
    // log.data = message + a + b + c +d
    // output: {message: abc, a: t, b: 100, c: "0xabcasdasdasd", d: "abcd"}
    // schema data
    // table: event_lowstr($eventName)_$contractID
    // table name: event_messagechanged_1
    // base columns: PK (id), contractId, mhKey, eventID
    // dynamic columns: message, a, b, c, d

Recommend Projects

React

A declarative, efficient, and flexible JavaScript library for building user interfaces.
Vue.js

🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
Typescript

TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
TensorFlow

An Open Source Machine Learning Framework for Everyone
Django

The Web framework for perfectionists with deadlines.
Laravel

A PHP framework for web artisans
D3

Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

javascript

JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
web

Some thing interesting about web. New door for the world.
server

A server is a program made to process requests and deliver data to clients.
Machine learning

Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Visualization

Some thing interesting about visualization, use data art
Game

Some thing interesting about game, make everyone happy.

Recommend Org

Facebook

We are working to build community through open source technology. NB: members must have two-factor auth.
Microsoft

Open source projects and samples from Microsoft.
Google

Google ❤️ Open Source for everyone.
Alibaba

Alibaba Open Source for everyone
D3

Data-Driven Documents codes.
Tencent

China tencent open source team.