Covid integration

.gitignore

@@ -60,3 +60,6 @@ typings/
 
 # next.js build output
 .next
+
+#adding model to gitignore
+saved_models/

DATA/NP/in_ext_loc.py

@@ -0,0 +1,25 @@
+import pudb
+import pickle
+f = open("IN.txt", "r")
+all_poses = {}
+
+count = 0
+while True:
+	count += 1
+	print(count)
+	line = f.readline()
+	if not line: break
+	names = [line.split("\t")[2]]
+	names.extend(line.split("\t")[3])
+	lat = line.split("\t")[4]
+	lon = line.split("\t")[4]
+	names = [x.lower() for x in names]
+	names = list(set(names))
+	for name in names:
+		if name not in all_poses:
+			all_poses[name] = []
+		all_poses[name].append((lat,lon))
+
+f.close()
+with open('IN_loc.p', 'wb') as handle:
+   pickle.dump(all_poses, handle)

app.py

@@ -21,6 +21,10 @@
 import sys
 import json
 from urllib.parse import unquote
+from classify_tweets_covid_infer import BertSentClassifier
+from classify_tweets_covid_infer import evaluate_bert
+
+# model = load_model()
 ps_stemmer= nltk.stem.porter.PorterStemmer()
 
 ## CORS
@@ -30,7 +34,7 @@
 
 # import en_core_web_sm
 # nlp = en_core_web_sm.load()
-nlp = spacy.load("en_core_web_sm")
+nlp=spacy.load('en')
 np_labels=set(['nsubj','dobj','pobj','iobj','conj','nsubjpass','appos','nmod','poss','parataxis','advmod','advcl'])
 subj_labels=set(['nsubj','nsubjpass','csubj','csubjpass'])
 modifiers=['nummod','compound','amod','punct']
@@ -614,13 +618,17 @@ def create_resource_list(text):
 	'''
 	return a,b,loc_list_2,modified_array,d, final_resource_dict
 
+def get_classification(text):
+	# global model
+	return evaluate_bert(text)
+
 bucket_classes=['shelter', 'food','medical','logistic']
 
 @app.route('/parse', methods=['GET', 'POST', 'OPTIONS'])
 @cross_origin()
 def parseResources():
 	global_resource_list={}
-	# print(request.body)
+	# print(flask.request.body)
 	resource, line = {}, ''
 	print(flask.request.json)
 	print(unquote(flask.request.query_string.decode('utf-8')))
@@ -713,12 +721,125 @@ def parseResources():
 	# print(class_list)
 	## Need to add quantity
 	## Ritam yaha dekh
-
+	resource['Classification'] = int(get_classification(line)[0])
 	# print('=>', resource['contact'], '\na=>', a, '\nb=>', b, '\nc=>', c, '\nm=>', modified_array, '\nd=>', d, '\nf=>', final_resource_dict)
 	# print(final_resource_dict)
 	print('Returning', resource)
 	return flask.jsonify(resource)
 
+
+@app.route('/parseStream', methods=['GET', 'POST', 'OPTIONS'])
+@cross_origin()
+def parseResourcesStream():
+	global_resource_list={}
+	# resource, line = {}, ''
+	resource_stream = []
+	print(flask.request.json)
+	# print(unquote(flask.request.query_string.decode('utf-8')))
+	if flask.request and flask.request.json and'text' in flask.request.json:
+		line_stream = flask.request.json['text']
+	# else:
+	# 	line = json.loads(unquote(flask.request.query_string.decode('utf-8')))['text']
+	for line in line_stream:
+		resource = {}
+		print('Received for parsing: ', line)
+		contacts = get_contact(line)
+		t2 = location.tweet_preprocess2(line,[])
+		sources,b,locations,modified_array,rWords, final_resource_dict  =create_resource_list(line)
+		# source_list,final_resource_keys,loc_list	,dup_final_resource_keys => post_process
+
+		## source_list, final_resource_keys, loc_list_2, modified_array?, dup_final_resource_keys, final_resource_dict?
+		# resource['x']=((line,a,b,c,modified_array,d, final_resource_dict))
+		resource["text"] = line
+		resource['Contact'] = {'Phone number': list(contacts[0]), "Email": list(contacts[1])}
+		resource['Sources'] = sources
+		resource['ResourceWords'] = rWords
+		resource['Locations'], resource['Resources'] = dict(), {}
+		# resource['Locations'] = locations
+		for each in locations:
+			# print(each[0], "<>", each[1])
+			resource['Locations'][each[0]] = {"long": float(each[1][1]), "lat": float(each[1][0])}
+		# f is Resources type
+		resources_bucket = {}
+
+		for each_resource in final_resource_dict:
+			buckets = final_resource_dict[each_resource]
+			assigned = False
+			for bucket in buckets:
+				if bucket in bucket_classes and not assigned:
+					if bucket not in resource['Resources']:
+						resource['Resources'][bucket] = {}
+					resource['Resources'][bucket][each_resource] = 'None'
+					assigned = True
+					resources_bucket[each_resource] = bucket
+
+
+		split_text= line.split()
+		class_list={}
+
+		for rWord in rWords:
+			s = {}
+			prev_words = [ split_text[i-1] for i in range(0,len(split_text)) if rWord.startswith(split_text[i]) ]
+			qt = 'None'
+
+			try:
+				for word in prev_words:
+					word=word.replace(',','')
+					if word.isnumeric()==True:
+						qt=str(word)
+						break
+					else:
+						try:
+							qt=str(w2n.word_to_num(word))
+							break
+						except Exception as e:	
+							continue
+
+				if qt=='None':	
+					elems=rWord.strip().split()	
+					word=elems[0]
+					rWord2=" ".join(elems[1:])
+
+					word=word.replace(',','')
+					if word.isnumeric()==True:
+						qt=str(word)
+					else:
+						try:
+							qt=str(w2n.word_to_num(word))
+						except Exception as e:
+							pass
+
+				if qt != 'None' and qt in rWord:
+					print(rWord, qt)
+					continue
+
+
+			except Exception as e:
+				exc_type, exc_obj, exc_tb = sys.exc_info()
+				fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+				print(exc_type, fname, exc_tb.tb_lineno)
+				qt='None'
+
+			# class_list[rWord]= qt
+			resource['Resources'][resources_bucket[rWord]][rWord] = qt
+		resource_stream.append(resource)
+		# print(class_list)
+		## Need to add quantity
+		## Ritam yaha dekh
+		# resource['Classification'] = get_classification(line_stream)
+	classification_stream = get_classification(line_stream)
+	print(classification_stream)
+	resource_stream_final = []
+	for i, cl in enumerate(classification_stream):
+		resource = resource_stream[i]
+		resource["Classification"] = int(cl)
+		resource_stream_final.append(resource)
+
+		# print('=>', resource['contact'], '\na=>', a, '\nb=>', b, '\nc=>', c, '\nm=>', modified_array, '\nd=>', d, '\nf=>', final_resource_dict)
+		# print(final_resource_dict)
+	print('Returning', resource_stream_final)
+	return flask.jsonify(resource_stream_final)
+
 # add routes for nodejs backend via here as well
 
 @app.route('/', methods=['GET', 'OPTIONS'])

app_covid.py

@@ -0,0 +1,113 @@
+import os
+import flask
+from flask import Flask
+app = Flask(__name__)
+
+
+import re
+import json
+from urllib.parse import unquote
+import location
+import pudb
+
+## CORS
+from flask_cors import CORS, cross_origin
+cors = CORS(app)
+app.config['CORS_HEADERS'] = 'Content-Type'
+
+tel_no="([+]?[0]?[1-9][0-9\s]*[-]?[0-9\s]+)"
+email="([a-zA-Z0-9]?[a-zA-Z0-9_.]+[@][a-zA-Z]+[.](com|net|edu|in|org|en))"
+http_url='http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\)]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+
+def get_contact(text):
+	contacts=[]
+	numbers=re.findall(tel_no,text)
+	temp=set()
+	for i in numbers:
+		if len(i.replace(' ',''))>=7:
+			temp.add(i)
+	contacts.append(temp)		
+	temp=set()
+	mails= re.findall(email,text)
+	for i in mails:
+		temp.add(i)
+	contacts.append(temp)	
+	temp=set()	
+	urls= re.findall(http_url,text)	
+	for i in urls:
+		temp.add(i)
+	contacts.append(temp)
+	return contacts	
+
+def get_classification(text, resource_list):
+    if "need" in text or "require" in text:
+        label = 0
+    elif "availab" in text or len(resource_list) != 0:
+        label = 1
+    else:
+        label = 2
+    return (label)
+
+resources = {
+    "oxygen": "Oxygen",
+    "o2": "Oxygen",
+    "ventilator": "Ventilator",
+    "bed": "Beds",
+    "icu": "Beds",
+    "remdes": "Remdesivir",
+    "plasma": "Plasma",
+    "consultation": "Doctor",
+    "ambulance": "Ambulance"
+}
+def get_location_covid(text):
+    text = text.lower()
+    places = location.return_location_list(text)
+    each_loc = [place[0] for place in places]
+    places_to_remove = []
+    resource_text = ""
+    for resource in resources:
+        if resource in each_loc:
+            places_to_remove.append(each_loc.index(resource))
+        if resource in text:
+            resource_text = resource_text+resources[resource]+" "
+    places_to_remove.sort(reverse=True)
+    for ptr in places_to_remove:
+        del places[ptr]
+    return resource_text, places
+
+@app.route('/parse', methods=['GET', 'POST', 'OPTIONS'])
+@cross_origin()
+def parseResources():
+    resource, line = {}, ''
+    print(flask.request.json)
+    print(unquote(flask.request.query_string.decode('utf-8')))
+    if flask.request and flask.request.json and'text' in flask.request.json:
+        line = flask.request.json['text']
+    else:
+        line = json.loads(unquote(flask.request.query_string.decode('utf-8')))['text']
+    print('Received for parsing: ', line)
+    contacts = get_contact(line)
+    resource_text, locations = get_location_covid(line)
+    print(resource_text,locations)
+    resource['Contact'] = {'Phone number': list(contacts[0]), "Email": list(contacts[1])}
+    resource['Sources'] = {}
+    resource['ResourceWords'] = resource_text.strip(" ").split(" ")
+    resource['Locations'], resource['Resources'] = dict(), {}
+    resource['Resources'] = {"resources": resource['ResourceWords']}
+    for each in locations:
+        resource['Locations'][each[0]] = {"long": float(each[1][0][1]), "lat": float(each[1][0][0])}
+    resource['Classification'] = int(get_classification(line, resource['ResourceWords']))
+    print('Returning', resource)
+    return flask.jsonify(resource)
+
+# add routes for nodejs backend via here as well
+@app.route('/', methods=['GET', 'OPTIONS'])
+@cross_origin()
+def base():
+	with open('index.html', 'r') as f:
+		txt = f.readlines()
+	return ''.join(txt)
+
+if __name__ == '__main__':
+	port = int(os.environ.get('PORT', 5000))
+	app.run(host='0.0.0.0', port=port, debug=True)