All logs are below, but the important part is that the list compute plan command returns:
substra list compute_plan
COMPUTE PLAN ID TRAINTUPLES COUNT COMPOSITE TRAINTUPLES COUNT AGGREGATETUPLES COUNT TESTTUPLES COUNT STATUS
e7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890 101 0 100 0 todo
But there should be 201 traintuples, not 101.
substra list traintuple
Requests error status 400: {"message":"could not retrieve parent traintuple with key 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf - traintuple 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf not found"}
Error: Request failed: InvalidRequest: 400 Client Error: Bad Request for url: http://substra-backend.node-1.com/traintuple/: could not retrieve parent traintuple with key 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf - traintuple 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf not found
pytest tests/test_load.py -rs -v --durations=0 -k test_load_multi_node_aggregates
==================================================================== test session starts ====================================================================
platform darwin -- Python 3.7.3, pytest-5.3.0, py-1.8.0, pluggy-0.13.0 -- /Users/jeremy/.virtualenvs/substra/bin/python3.7
cachedir: .pytest_cache
tests run uuid: 5390c4c731ce40bb9d8ecc638e402b9d'
substra network configuration loaded from: '/Users/jeremy/substra/substra-tests/tests/../values.yaml'
substra network setup:
- node: name=node-1 msp_id=MyOrg1MSP address=http://substra-backend.node-1.com
- node: name=node-2 msp_id=MyOrg2MSP address=http://substra-backend.node-2.com
rootdir: /Users/jeremy/substra/substra-tests
plugins: celery-4.2.1, mock-1.12.1, cov-2.8.1
collected 3 items / 2 deselected / 1 selected
tests/test_load.py::test_load_multi_node_aggregates[100] FAILED [100%]
========================================================================= FAILURES ==========================================================================
___________________________________________________________ test_load_multi_node_aggregates[100] ____________________________________________________________
self = <substra.sdk.rest_client.Client object at 0x10ecb3668>, request_name = 'get', url = 'http://substra-backend.node-1.com/traintuple/'
request_kwargs = {'params': 'search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890'}
fn = <function get at 0x10da28c80>
kwargs = {'params': 'search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890'}, r = <Response [400]>
def _request(self, request_name, url, **request_kwargs):
"""Base request helper."""
if request_name == 'get':
fn = requests.get
elif request_name == 'post':
fn = requests.post
else:
raise NotImplementedError
# override default request arguments with input arguments
kwargs = dict(self._default_kwargs)
kwargs.update(request_kwargs)
# do HTTP request and catch generic exceptions
try:
r = fn(url, headers=self._headers, **kwargs)
> r.raise_for_status()
../substra/substra/sdk/rest_client.py:107:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <Response [400]>
def raise_for_status(self):
"""Raises stored :class:`HTTPError`, if one occurred."""
http_error_msg = ''
if isinstance(self.reason, bytes):
# We attempt to decode utf-8 first because some servers
# choose to localize their reason strings. If the string
# isn't utf-8, we fall back to iso-8859-1 for all other
# encodings. (See PR #3538)
try:
reason = self.reason.decode('utf-8')
except UnicodeDecodeError:
reason = self.reason.decode('iso-8859-1')
else:
reason = self.reason
if 400 <= self.status_code < 500:
http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url)
elif 500 <= self.status_code < 600:
http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url)
if http_error_msg:
> raise HTTPError(http_error_msg, response=self)
E requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://substra-backend.node-1.com/traintuple/?search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890
../../.virtualenvs/substra/lib/python3.7/site-packages/requests/models.py:940: HTTPError
During handling of the above exception, another exception occurred:
compute_plan_size = 100
global_execution_env = (<substratest.factory.AssetsFactory object at 0x10db53e48>, Network(sessions=[<substratest.client.Session object at 0x10db53d68>, <substratest.client.Session object at 0x10db24be0>]))
@pytest.mark.parametrize('compute_plan_size', COMPUTE_PLAN_SIZES)
def test_load_multi_node_aggregates(compute_plan_size, global_execution_env):
"""
Shape of the compute plan:
Node A: traintuple 0 --> traintuple --> aggregate --> traintuple --> aggregate ...
Node B: \-> traintuple / \-> traintuple /
"""
factory, network = global_execution_env
session_1 = network.sessions[0].copy()
session_2 = network.sessions[0].copy()
dataset_1 = session_1.state.datasets[0]
dataset_2 = session_2.state.datasets[0]
spec = factory.create_algo()
algo = session_1.add_algo(spec)
spec = factory.create_aggregate_algo()
aggregate_algo = session_1.add_aggregate_algo(spec)
cp_spec = factory.create_compute_plan()
first_tuple = cp_spec.add_traintuple(
algo=algo,
dataset=dataset_1,
data_samples=[dataset_1.train_data_sample_keys[0]],
)
previous_aggregatetuple = first_tuple
for _ in range(compute_plan_size):
tuples = [
cp_spec.add_traintuple(
algo=algo,
dataset=dataset,
data_samples=[dataset.train_data_sample_keys[0]],
in_models=[previous_aggregatetuple],
) for dataset in [dataset_1, dataset_2]
]
previous_aggregatetuple = cp_spec.add_aggregatetuple(
aggregate_algo=aggregate_algo,
worker=session_1.node_id,
in_models=tuples,
)
cp = session_1.add_compute_plan(cp_spec)
> first_tuple = cp.list_traintuple()[0]
tests/test_load.py:150:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
substratest/assets.py:422: in list_traintuple
tuples = self._session.list_traintuple(filters=filters)
substratest/client.py:216: in list_traintuple
res = self._client.list_traintuple(*args, **kwargs)
../substra/substra/sdk/client.py:628: in list_traintuple
return self.client.list(assets.TRAINTUPLE, filters=filters)
../substra/substra/sdk/rest_client.py:184: in list
**request_kwargs,
../substra/substra/sdk/rest_client.py:155: in request
**request_kwargs,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <substra.sdk.rest_client.Client object at 0x10ecb3668>, request_name = 'get', url = 'http://substra-backend.node-1.com/traintuple/'
request_kwargs = {'params': 'search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890'}
fn = <function get at 0x10da28c80>
kwargs = {'params': 'search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890'}, r = <Response [400]>
def _request(self, request_name, url, **request_kwargs):
"""Base request helper."""
if request_name == 'get':
fn = requests.get
elif request_name == 'post':
fn = requests.post
else:
raise NotImplementedError
# override default request arguments with input arguments
kwargs = dict(self._default_kwargs)
kwargs.update(request_kwargs)
# do HTTP request and catch generic exceptions
try:
r = fn(url, headers=self._headers, **kwargs)
r.raise_for_status()
except requests.exceptions.ConnectionError as e:
raise exceptions.ConnectionError.from_request_exception(e)
except requests.exceptions.Timeout as e:
raise exceptions.Timeout.from_request_exception(e)
except requests.exceptions.HTTPError as e:
logger.error(f"Requests error status {e.response.status_code}: {e.response.text}")
if e.response.status_code == 400:
> raise exceptions.InvalidRequest.from_request_exception(e)
E substra.sdk.exceptions.InvalidRequest: 400 Client Error: Bad Request for url: http://substra-backend.node-1.com/traintuple/?search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890: could not retrieve parent traintuple with key 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf - traintuple 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf not found
../substra/substra/sdk/rest_client.py:119: InvalidRequest
--------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------
ERROR substra.sdk.rest_client:rest_client.py:116 Requests error status 400: {"message":"could not retrieve parent traintuple with key 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf - traintuple 40a30edeb7313f0e9acf7e04a86d6af5a3e13123623b70014d49eebe3a11beaf not found"}
================================================================== slowest test durations ===================================================================
8.45s call tests/test_load.py::test_load_multi_node_aggregates[100]
7.48s setup tests/test_load.py::test_load_multi_node_aggregates[100]
0.01s teardown tests/test_load.py::test_load_multi_node_aggregates[100]
============================================================= 1 failed, 2 deselected in 16.27s ==============================================================
│ substra-backend [SQL Queries for] /compute_plan/ │
│ │
│ substra-backend [0.003] SELECT authtoken_token.key, authtoken_token.user_id, authtoken_token.created, auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.e │
│ mail, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FROM authtoken_token INNER JOIN auth_user ON (authtoken_token.user_id = auth_user.id) WHERE authtoken_token.key = '0b5fc165082e18cb851571373cb127b611b2be58' │
│ │
│ substra-backend [TOTAL TIME: 0.003 seconds (1 queries)] │
│ substra-backend [19/Dec/2019 15:25:22] "POST /compute_plan/ HTTP/1.1" 201 20359 │
│ │
│ │
│ substra-backend [SQL Queries for] /traintuple/ │
│ │
│ substra-backend [0.003] SELECT authtoken_token.key, authtoken_token.user_id, authtoken_token.created, auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.e │
│ mail, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FROM authtoken_token INNER JOIN auth_user ON (authtoken_token.user_id = auth_user.id) WHERE authtoken_token.key = '0b5fc165082e18cb851571373cb127b611b2be58' │
│ │
│ substra-backend [TOTAL TIME: 0.003 seconds (1 queries)] │
│ substra-backend [19/Dec/2019 15:25:22] "GET /traintuple/?search=traintuple%3AcomputePlanID%3Ae7f07657336f3b0841b8cf03369dab3b86a9e0edbbf803970cc7e0806e9ba890 HTTP/1.1" 400 212 │
│ │
│ │
│ substra-backend [SQL Queries for] /algo/aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff/file/ │
│ │
│ substra-backend [0.002] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.002] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.019] SELECT substrapp_algo.pkhash, substrapp_algo.file, substrapp_algo.description, substrapp_algo.validated FROM substrapp_algo WHERE substrapp_algo.pkhash = 'aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff' │
│ │
│ substra-backend [TOTAL TIME: 0.023 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:23] "GET /algo/aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff/file/ HTTP/1.1" 200 548 │
│ substra-backend [19/Dec/2019 15:25:31] "GET /readiness HTTP/1.1" 200 2 │
│ substra-backend INFO - 2019-12-19 15:25:34,062 - events.apps - Processing task da0f0318fe6262e5911058f69c754ab617831ea3004281aa95708e94456a8392: type=traintuple status=todo │
│ │
│ │
│ substra-backend [SQL Queries for] /algo/aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff/file/ │
│ │
│ substra-backend [0.003] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT substrapp_algo.pkhash, substrapp_algo.file, substrapp_algo.description, substrapp_algo.validated FROM substrapp_algo WHERE substrapp_algo.pkhash = 'aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff' │
│ │
│ substra-backend [TOTAL TIME: 0.005 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:34] "GET /algo/aeb2d551a1fb776e61d1d79e61ba72b1ae39a521e579416437445ff2743a53ff/file/ HTTP/1.1" 200 548 │
│
substra-backend [SQL Queries for] /model/e41b28842f09293ea616a056cfde1f9da1ac550135802efa346f15b576b179ca/file/ │
│ │
│ substra-backend [0.002] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT substrapp_model.pkhash, substrapp_model.validated, substrapp_model.file FROM substrapp_model WHERE substrapp_model.pkhash = 'e41b28842f09293ea616a056cfde1f9da1ac550135802efa346f15b576b179ca' │
│ │
│ substra-backend [TOTAL TIME: 0.004 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:35] "GET /model/e41b28842f09293ea616a056cfde1f9da1ac550135802efa346f15b576b179ca/file/ HTTP/1.1" 200 6 │
│ substra-backend INFO - 2019-12-19 15:25:39,369 - events.apps - Processing task 5384f339553b3f594235fe51acfa80e8475aeb97a93dee05ea9a44103939b238: type=aggregatetuple status=todo │
│ │
│ │
│ substra-backend [SQL Queries for] /aggregate_algo/e93f5354ad4470d15ee7418af5a1263b54acc3a05460ad583c051cb2b235dc4c/file/ │
│ │
│ substra-backend [0.002] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT substrapp_aggregatealgo.pkhash, substrapp_aggregatealgo.file, substrapp_aggregatealgo.description, substrapp_aggregatealgo.validated FROM substrapp_aggregatealgo WHERE substrapp_aggregatealgo.pkhash = 'e93f5354ad4470d15ee7418af5a126 │
│ 3b54acc3a05460ad583c051cb2b235dc4c' │
│ │
│ substra-backend [TOTAL TIME: 0.004 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:39] "GET /aggregate_algo/e93f5354ad4470d15ee7418af5a1263b54acc3a05460ad583c051cb2b235dc4c/file/ HTTP/1.1" 200 529 │
│ │
│ │
│ substra-backend [SQL Queries for] /model/2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b/file/ │
│ │
│ substra-backend [0.002] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT substrapp_model.pkhash, substrapp_model.validated, substrapp_model.file FROM substrapp_model WHERE substrapp_model.pkhash = '2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b' │
│ │
│ substra-backend [TOTAL TIME: 0.004 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:40] "GET /model/2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b/file/ HTTP/1.1" 200 6 │
substra-backend [SQL Queries for] /model/2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b/file/ │
│ │
│ substra-backend [0.002] SELECT auth_user.id, auth_user.password, auth_user.last_login, auth_user.is_superuser, auth_user.username, auth_user.first_name, auth_user.last_name, auth_user.email, auth_user.is_staff, auth_user.is_active, auth_user.date_joined FR │
│ OM auth_user WHERE auth_user.username = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT node_incomingnode.node_id, node_incomingnode.secret FROM node_incomingnode WHERE node_incomingnode.node_id = 'MyOrg1MSP' │
│ │
│ substra-backend [0.001] SELECT substrapp_model.pkhash, substrapp_model.validated, substrapp_model.file FROM substrapp_model WHERE substrapp_model.pkhash = '2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b' │
│ │
│ substra-backend [TOTAL TIME: 0.004 seconds (3 queries)] │
│ substra-backend [19/Dec/2019 15:25:40] "GET /model/2302f37e01ff8fb57a22e7c5f3346b551997bd6dc6e69e2f6e5725bff263cf6b/file/ HTTP/1.1" 200 6 │
│ substra-backend INFO - 2019-12-19 15:25:45,051 - events.apps - Processing task f170eb58c5b5932e3e9c56c0004d3c1f0755b46b96f58b68e3a9a87ec69e04cc: type=traintuple status=todo │
│ substra-backend Process Process-2: │
│ substra-backend Traceback (most recent call last): │
│ substra-backend File "/usr/local/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap │
│ substra-backend self.run() │
│ substra-backend File "/usr/local/lib/python3.6/multiprocessing/process.py", line 93, in run │
│ substra-backend self._target(*self._args, **self._kwargs) │
│ substra-backend File "/usr/src/app/events/apps.py", line 138, in wait │
│ substra-backend loop.run_until_complete(stream) │
│ substra-backend File "/usr/local/lib/python3.6/asyncio/base_events.py", line 484, in run_until_complete │
│ substra-backend return future.result() │
│ substra-backend File "/usr/local/lib/python3.6/site-packages/hfc/fabric/channel/channel_eventhub.py", line 560, in handle_stream │
│ substra-backend self._processChaincodeEvents(block) │
│ substra-backend File "/usr/local/lib/python3.6/site-packages/hfc/fabric/channel/channel_eventhub.py", line 465, in _processChaincodeEvents │
│ substra-backend e['tx_status']) │
│ substra-backend File "/usr/src/app/events/apps.py", line 72, in on_tuples │
│ substra-backend tuple_owner = tuple_get_worker(tuple_type, _tuple) │
│ substra-backend File "/usr/src/app/events/apps.py", line 43, in tuple_get_worker │
│ substra-backend return _tuple['dataset']['worker'] │
│ substra-backend TypeError: 'NoneType' object is not subscriptable │