wlmwng / inca Goto Github PK

try:
    rules_fox = [
        {"regexp": "\\n[A-Z0-9 :,\\'!@\$\(\)\-\.\?\:\;]+(?:\\n|$)", "replace_with": ""},
        {
            "regexp": "Get all the latest news on coronavirus and more delivered daily to your inbox\. Sign up here",
            "replace_with": "",
        },
    ]

    # generator
    docs_regexp = myinca.processing.multireplace(
        docs_or_query="foxnews",
        field="article_maintext",
        rules=rules_fox,
        save=True,
        new_key="article_maintext_0",
        action="batch",
        batchsize=100,
        
    )
    for doc in docs_regexp:
        # runs process on doc
        # doesn't yield updated doc since it saves to db
        pass

except Exception as e:
    LOGGER.warning(e)

2021-11-02 07:42:55,030 - [INFO] - INCA - (processor_class.py)._doctype_query_or_list(228) - assuming documents of given type should be processed
2021-11-02 07:42:55,030 - [INFO] - INCA - (processor_class.py)._doctype_query_or_list(234) - force=False, ignoring documents where the result key exists (and has non-NULL value)
2021-11-02 07:42:55,030 - [WARNING] - main - (1275500693.py).<module>(27) - name 'bulksize' is not defined

    def runwrap(
        self,
        docs_or_query,
        field,
        new_key=None,
        save=False,
        force=False,
        action="run",
        *args,
        **kwargs
    ):

    ...
        elif action == "batch":
            for num, batch in enumerate(_batcher(documents, batchsize=bulksize)):
                core.database.bulk_upsert().run(
                    documents=[
                        target_func.run(
                            document=doc, field=field, force=force, *args, **kwargs
                        )
                        for doc in batch
                    ]
                )
                now = datetime.datetime.now()
                logger.info("processed batch {num} {now}".format(**locals()))
                yield batch