Comments (4)
@UnderJollyRoger, could you send the code you are using to call the servicfunctions,py? send_email is a .py in the same directory as the servicefunctions.py so we just need to make sure it is in the import path.
from crowdsource-reporter-scripts.
`from send_email import EmailServer
import re
from datetime import datetime as dt
from os import path, sys
from arcgis.gis import GIS
from arcgis.features import FeatureLayer
import json
#id_settings = {}
#modlists = {}
def _add_message(msg, ertype='ERROR'):
print("{}: {}".format(ertype, msg))
with open(path.join(sys.path[0], 'id_log.log'), 'a') as log:
log.write("{} -- {}: {}".format(dt.now(), ertype, msg))
return
def _report_failures(results):
for result in results['updateResults']:
if not result['success']:
_add_message('{}: {}'.format(result['error']['code'], result['error']['description']))
return
def _get_features(feature_layer, where_clause, return_geometry=False):
"""Get the features for the given feature layer of a feature service. Returns a list of json features.
Keyword arguments:
feature_layer - The feature layer to return the features for
where_clause - The expression used in the query"""
total_features = []
max_record_count = feature_layer.properties['maxRecordCount']
if max_record_count < 1:
max_record_count = 1000
offset = 0
while True:
if not where_clause:
where_clause = "1=1"
features = feature_layer.query(where=where_clause,
return_geometry=return_geometry,
result_offset=offset,
result_record_count=max_record_count).features
total_features += features
if len(features) < max_record_count:
break
offset += len(features)
return total_features
def add_identifiers(lyr, seq, fld):
"""Update features in an agol/portal service with id values
Return next valid sequence value"""
# Get features without id
value = id_settings[seq]['next value']
fmt = id_settings[seq]['pattern']
interval = id_settings[seq]['interval']
rows = _get_features(lyr, """{} is null""".format(fld))
# For each feature, update id, and increment sequence value
for row in rows:
row.attributes[fld] = fmt.format(value)
value += interval
if rows:
results = lyr.edit_features(updates=rows)
_report_failures(results)
return value
def enrich_layer(source, target, settings):
wkid = target.properties.extent.spatialReference.wkid
# Query for target features
rows = _get_features(target, "1=1", return_geometry=True)
for row in rows:
# Perform spatial query to get attributes of intersecting feature
ptgeom = {'geometry': row.geometry,
'spatialRel': 'esriSpatialRelIntersects',
'geometryType': 'esriGeometryPoint',
'inSR': wkid
}
sourcefeat = source.query(geometry_filter=ptgeom)
try:
# Only first feature is processed
source_val = sourcefeat.features[0].attributes[settings['source']]
row.attributes[settings['target']] = source_val
except IndexError:
continue # no intersecting feature found
if rows:
results = target.edit_features(updates=rows)
_report_failures(results)
return
def build_expression(words, match_type, subs):
"""Build an all-caps regular expression for matching either exact or
partial strings"""
re_string = ''
for word in words:
new_word = ''
for char in word.upper():
# If listed, include substitution characters
if char in subs.keys():
new_word += "[" + char + subs[char] + "]"
else:
new_word += "[" + char + "]"
# Filter using only exact matches of the string
if match_type == 'EXACT':
re_string += '\\b{}\\b|'.format(new_word)
# Filter using all occurances of the letter combinations specified
else:
re_string += '.*{}.*|'.format(new_word)
# Last character will always be | and must be dropped
return re_string[:-1]
def moderate_features(lyr, settings):
rows = _get_features(lyr, settings['sql'])
for row in rows:
for field in settings['scan fields'].split(';'):
try:
text = row.get_value(field)
text = text.upper()
except AttributeError: # Handles empty fields
continue
if re.search(modlists[settings['list']], text):
row.attributes[settings['field']] = settings['value']
break
if rows:
results = lyr.edit_features(updates=rows)
_report_failures(results)
return
def _get_value(row, fields, sub):
val = row.attributes[sub]
if val is None:
val = ''
elif type(val) != str:
for field in fields:
if field['name'] == sub and 'Date' in field['type']:
try:
val = dt.fromtimestamp(
row.attributes[sub]).strftime('%c')
except OSError: # timestamp in milliseconds
val = dt.fromtimestamp(
row.attributes[sub] / 1000).strftime('%c')
break
else:
val = str(val)
return val
def build_email(row, fields, settings):
email_subject = ''
email_body = ''
if settings['recipient'] in row.fields:
email = row.attributes[settings['recipient']]
else:
email = settings['recipient']
try:
html = path.join(path.dirname(__file__), settings['template'])
with open(html) as file:
email_body = file.read()
email_subject = settings['subject']
if substitutions:
for sub in substitutions:
if sub[1] in row.fields:
val = _get_value(row, fields, sub[1])
email_body = email_body.replace(sub[0], val)
email_subject = email_subject.replace(sub[0], val)
else:
email_body = email_body.replace(sub[0], str(sub[1]))
email_subject = email_subject.replace(sub[0], str(sub[1]))
except:
_add_message('Failed to read email template {}'.format(html))
return email, email_subject, email_body
def main(configuration_file):
try:
with open(configuration_file) as configfile:
cfg = json.load(configfile)
gis = GIS(cfg['organization url'], cfg['username'], cfg['password'])
# Get general id settings
global id_settings
id_settings = {}
for option in cfg['id sequences']:
id_settings[option['name']] = {'interval': int(option['interval']),
'next value': int(option['next value']),
'pattern': option['pattern']}
# Get general moderation settings
global modlists
modlists = {}
subs = cfg['moderation settings']['substitutions']
for modlist in cfg['moderation settings']['lists']:
words = [str(word).upper().strip() for word in modlist['words'].split(',')]
modlists[modlist['filter name']] = build_expression(words, modlist['filter type'], subs)
# Get general email settings
server = cfg['email settings']['smtp server']
username = cfg['email settings']['smtp username']
password = cfg['email settings']['smtp password']
tls = cfg['email settings']['use tls']
from_address = cfg['email settings']['from address']
if not from_address:
from_address = ''
reply_to = cfg['email settings']['reply to']
if not reply_to:
reply_to = ''
global substitutions
substitutions = cfg['email settings']['substitutions']
# Process each service
for service in cfg['services']:
try:
lyr = FeatureLayer(service['url'], gis=gis)
# GENERATE IDENTIFIERS
idseq = service['id sequence']
idfld = service['id field']
if id_settings and idseq and idfld:
if idseq in id_settings:
new_sequence_value = add_identifiers(lyr, idseq, idfld)
id_settings[idseq]['next value'] = new_sequence_value
else:
_add_message('Sequence {} not found in sequence settings'.format(idseq), 'WARNING')
# ENRICH REPORTS
if service['enrichment']:
# reversed, sorted list of enrichment settings
enrich_settings = sorted(service['enrichment'], key=lambda k: k['priority'], reverse=True)
for reflayer in enrich_settings:
source_features = FeatureLayer(reflayer['url'], gis)
enrich_layer(source_features, lyr, reflayer)
# MODERATION
if modlists:
for query in service['moderation']:
if query['list'] in modlists:
moderate_features(lyr, query)
else:
_add_message('Moderation list {} not found in moderation settings'.format(modlist), 'WARNING')
# SEND EMAILS
if service['email']:
with EmailServer(server, username, password, tls) as email_server:
for message in service['email']:
rows = _get_features(lyr, message['sql'])
for row in rows:
address, subject, body = build_email(row, lyr.properties.fields, message)
if address and subject and body:
try:
email_server.send(from_address=from_address,
reply_to=reply_to,
to_addresses=[address],
subject=subject,
email_body=body)
row.attributes[message['field']] = message['sent value']
except:
_add_message('email failed to send for feature {} in layer {}'.format(row.attributes, service['url']))
if rows:
results = lyr.edit_features(updates=rows)
_report_failures(results)
except Exception as ex:
_add_message('Failed to process service {}\n{}'.format(service['url'], ex))
except Exception as ex:
_add_message('Failed. Please verify all configuration values\n{}'.format(ex))
finally:
new_sequences = [{'name': seq,
'interval': id_settings[seq]['interval'],
'next value': id_settings[seq]['next value'],
'pattern': id_settings[seq]['pattern']} for seq in id_settings]
if not new_sequences == cfg['id sequences']:
cfg['id sequences'] = new_sequences
try:
with open(configuration_file, 'w') as configfile:
json.dump(cfg, configfile)
except Exception as ex:
_add_message('Failed to save identifier configuration values.\n{}\nOld values:{}\nNew values:{}'.format(ex, cfg['id sequences'], new_sequences))
if name == 'main':
main(path.join(path.dirname(file), 'servicefunctions.json'))`
from crowdsource-reporter-scripts.
@UnderJollyRoger, you won't be able to copy/paste the contents of the file into a python notebook and run it. There are a few sections in the script that are expecting to find files in the same directory as the original script that are not resolved because the notebook is executing from a different location. For example the send_email module is a .py in the same directory as the servicefunctions.py. In addition, the last line is looking for the servicefunctions.json in the same directory as well. I will continue to work with you on why it is failing from command line in the geonet post.
from crowdsource-reporter-scripts.
I am having the same issue and do not understand what has been done to solve the issue.
from crowdsource-reporter-scripts.
Related Issues (20)
- Send Email parameter help HOT 1
- add message to cityworks tool
- issue selecting a table from the current map HOT 9
- BUG-000113656 The documentation for Send Email Notifications Scripts do not say the tools do not send emails. HOT 1
- BUG-000113756 The ServiceFunction.py script produces errors indicating it could not read the email template when no substitution is used in the email. HOT 2
- ENH-000114300 Allow apps that use crowd source reporter to moderate comments. HOT 1
- ENH-000116673 Provide the ability to enter multiple email addresses in the Recipient Email Address section when using the Send Emails script found within the ServiceFunctions toolbox. HOT 10
- confirm order of operations for email
- add support for domains
- sql parameters
- enrich tool: preserve enrichment layer & target field
- issue with subbing values not from fields
- reduce connections to email server to 1/script run
- Connect2Cityworks.pyt KeyError operationalLayers HOT 3
- Remove calls to _get_features() function and replace with Python API's Query call
- Update any edit_features()/query() call to ONLY retrieve/send edits for needed attributes
- Enrich is currently query all features which can really slow when there are many rows in the layer
- Crowdsource to Cityworks integration error: json.decoder.JSONDecodeError HOT 2
- One of maps wouldn't open on a mobile HOT 4
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from crowdsource-reporter-scripts.