diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..aaedb43 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index ea5e44d..44a1e90 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ typings/ # next.js build output .next + +#adding model to gitignore +saved_models/ diff --git a/DATA/NP/IN_loc.p b/DATA/NP/IN_loc.p new file mode 100644 index 0000000..ba487ed Binary files /dev/null and b/DATA/NP/IN_loc.p differ diff --git a/DATA/NP/in_ext_loc.py b/DATA/NP/in_ext_loc.py new file mode 100644 index 0000000..bd806b3 --- /dev/null +++ b/DATA/NP/in_ext_loc.py @@ -0,0 +1,25 @@ +import pudb +import pickle +f = open("IN.txt", "r") +all_poses = {} + +count = 0 +while True: + count += 1 + print(count) + line = f.readline() + if not line: break + names = [line.split("\t")[2]] + names.extend(line.split("\t")[3]) + lat = line.split("\t")[4] + lon = line.split("\t")[4] + names = [x.lower() for x in names] + names = list(set(names)) + for name in names: + if name not in all_poses: + all_poses[name] = [] + all_poses[name].append((lat,lon)) + +f.close() +with open('IN_loc.p', 'wb') as handle: + pickle.dump(all_poses, handle) \ No newline at end of file diff --git a/__pycache__/classify_tweets_covid_infer.cpython-38.pyc b/__pycache__/classify_tweets_covid_infer.cpython-38.pyc new file mode 100644 index 0000000..9a065cc Binary files /dev/null and b/__pycache__/classify_tweets_covid_infer.cpython-38.pyc differ diff --git a/__pycache__/location_2.cpython-38.pyc b/__pycache__/location_2.cpython-38.pyc new file mode 100644 index 0000000..3f83440 Binary files /dev/null and b/__pycache__/location_2.cpython-38.pyc differ diff --git a/app.py b/app.py index 330d03b..88926b4 100755 --- a/app.py +++ b/app.py @@ -21,6 +21,10 @@ import sys import json from urllib.parse import unquote +from classify_tweets_covid_infer import BertSentClassifier +from classify_tweets_covid_infer import evaluate_bert + +# model = load_model() ps_stemmer= nltk.stem.porter.PorterStemmer() ## CORS @@ -30,7 +34,7 @@ # import en_core_web_sm # nlp = en_core_web_sm.load() -nlp = spacy.load("en_core_web_sm") +nlp=spacy.load('en') np_labels=set(['nsubj','dobj','pobj','iobj','conj','nsubjpass','appos','nmod','poss','parataxis','advmod','advcl']) subj_labels=set(['nsubj','nsubjpass','csubj','csubjpass']) modifiers=['nummod','compound','amod','punct'] @@ -614,13 +618,17 @@ def create_resource_list(text): ''' return a,b,loc_list_2,modified_array,d, final_resource_dict +def get_classification(text): + # global model + return evaluate_bert(text) + bucket_classes=['shelter', 'food','medical','logistic'] @app.route('/parse', methods=['GET', 'POST', 'OPTIONS']) @cross_origin() def parseResources(): global_resource_list={} - # print(request.body) + # print(flask.request.body) resource, line = {}, '' print(flask.request.json) print(unquote(flask.request.query_string.decode('utf-8'))) @@ -713,12 +721,125 @@ def parseResources(): # print(class_list) ## Need to add quantity ## Ritam yaha dekh - + resource['Classification'] = int(get_classification(line)[0]) # print('=>', resource['contact'], '\na=>', a, '\nb=>', b, '\nc=>', c, '\nm=>', modified_array, '\nd=>', d, '\nf=>', final_resource_dict) # print(final_resource_dict) print('Returning', resource) return flask.jsonify(resource) + +@app.route('/parseStream', methods=['GET', 'POST', 'OPTIONS']) +@cross_origin() +def parseResourcesStream(): + global_resource_list={} + # resource, line = {}, '' + resource_stream = [] + print(flask.request.json) + # print(unquote(flask.request.query_string.decode('utf-8'))) + if flask.request and flask.request.json and'text' in flask.request.json: + line_stream = flask.request.json['text'] + # else: + # line = json.loads(unquote(flask.request.query_string.decode('utf-8')))['text'] + for line in line_stream: + resource = {} + print('Received for parsing: ', line) + contacts = get_contact(line) + t2 = location.tweet_preprocess2(line,[]) + sources,b,locations,modified_array,rWords, final_resource_dict =create_resource_list(line) + # source_list,final_resource_keys,loc_list ,dup_final_resource_keys => post_process + + ## source_list, final_resource_keys, loc_list_2, modified_array?, dup_final_resource_keys, final_resource_dict? + # resource['x']=((line,a,b,c,modified_array,d, final_resource_dict)) + resource["text"] = line + resource['Contact'] = {'Phone number': list(contacts[0]), "Email": list(contacts[1])} + resource['Sources'] = sources + resource['ResourceWords'] = rWords + resource['Locations'], resource['Resources'] = dict(), {} + # resource['Locations'] = locations + for each in locations: + # print(each[0], "<>", each[1]) + resource['Locations'][each[0]] = {"long": float(each[1][1]), "lat": float(each[1][0])} + # f is Resources type + resources_bucket = {} + + for each_resource in final_resource_dict: + buckets = final_resource_dict[each_resource] + assigned = False + for bucket in buckets: + if bucket in bucket_classes and not assigned: + if bucket not in resource['Resources']: + resource['Resources'][bucket] = {} + resource['Resources'][bucket][each_resource] = 'None' + assigned = True + resources_bucket[each_resource] = bucket + + + split_text= line.split() + class_list={} + + for rWord in rWords: + s = {} + prev_words = [ split_text[i-1] for i in range(0,len(split_text)) if rWord.startswith(split_text[i]) ] + qt = 'None' + + try: + for word in prev_words: + word=word.replace(',','') + if word.isnumeric()==True: + qt=str(word) + break + else: + try: + qt=str(w2n.word_to_num(word)) + break + except Exception as e: + continue + + if qt=='None': + elems=rWord.strip().split() + word=elems[0] + rWord2=" ".join(elems[1:]) + + word=word.replace(',','') + if word.isnumeric()==True: + qt=str(word) + else: + try: + qt=str(w2n.word_to_num(word)) + except Exception as e: + pass + + if qt != 'None' and qt in rWord: + print(rWord, qt) + continue + + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + print(exc_type, fname, exc_tb.tb_lineno) + qt='None' + + # class_list[rWord]= qt + resource['Resources'][resources_bucket[rWord]][rWord] = qt + resource_stream.append(resource) + # print(class_list) + ## Need to add quantity + ## Ritam yaha dekh + # resource['Classification'] = get_classification(line_stream) + classification_stream = get_classification(line_stream) + print(classification_stream) + resource_stream_final = [] + for i, cl in enumerate(classification_stream): + resource = resource_stream[i] + resource["Classification"] = int(cl) + resource_stream_final.append(resource) + + # print('=>', resource['contact'], '\na=>', a, '\nb=>', b, '\nc=>', c, '\nm=>', modified_array, '\nd=>', d, '\nf=>', final_resource_dict) + # print(final_resource_dict) + print('Returning', resource_stream_final) + return flask.jsonify(resource_stream_final) + # add routes for nodejs backend via here as well @app.route('/', methods=['GET', 'OPTIONS']) diff --git a/app_covid.py b/app_covid.py new file mode 100644 index 0000000..3b34e39 --- /dev/null +++ b/app_covid.py @@ -0,0 +1,113 @@ +import os +import flask +from flask import Flask +app = Flask(__name__) + + +import re +import json +from urllib.parse import unquote +import location +import pudb + +## CORS +from flask_cors import CORS, cross_origin +cors = CORS(app) +app.config['CORS_HEADERS'] = 'Content-Type' + +tel_no="([+]?[0]?[1-9][0-9\s]*[-]?[0-9\s]+)" +email="([a-zA-Z0-9]?[a-zA-Z0-9_.]+[@][a-zA-Z]+[.](com|net|edu|in|org|en))" +http_url='http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\)]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' + +def get_contact(text): + contacts=[] + numbers=re.findall(tel_no,text) + temp=set() + for i in numbers: + if len(i.replace(' ',''))>=7: + temp.add(i) + contacts.append(temp) + temp=set() + mails= re.findall(email,text) + for i in mails: + temp.add(i) + contacts.append(temp) + temp=set() + urls= re.findall(http_url,text) + for i in urls: + temp.add(i) + contacts.append(temp) + return contacts + +def get_classification(text, resource_list): + if "need" in text or "require" in text: + label = 0 + elif "availab" in text or len(resource_list) != 0: + label = 1 + else: + label = 2 + return (label) + +resources = { + "oxygen": "Oxygen", + "o2": "Oxygen", + "ventilator": "Ventilator", + "bed": "Beds", + "icu": "Beds", + "remdes": "Remdesivir", + "plasma": "Plasma", + "consultation": "Doctor", + "ambulance": "Ambulance" +} +def get_location_covid(text): + text = text.lower() + places = location.return_location_list(text) + each_loc = [place[0] for place in places] + places_to_remove = [] + resource_text = "" + for resource in resources: + if resource in each_loc: + places_to_remove.append(each_loc.index(resource)) + if resource in text: + resource_text = resource_text+resources[resource]+" " + places_to_remove.sort(reverse=True) + for ptr in places_to_remove: + del places[ptr] + return resource_text, places + +@app.route('/parse', methods=['GET', 'POST', 'OPTIONS']) +@cross_origin() +def parseResources(): + resource, line = {}, '' + print(flask.request.json) + print(unquote(flask.request.query_string.decode('utf-8'))) + if flask.request and flask.request.json and'text' in flask.request.json: + line = flask.request.json['text'] + else: + line = json.loads(unquote(flask.request.query_string.decode('utf-8')))['text'] + print('Received for parsing: ', line) + contacts = get_contact(line) + resource_text, locations = get_location_covid(line) + print(resource_text,locations) + resource['Contact'] = {'Phone number': list(contacts[0]), "Email": list(contacts[1])} + resource['Sources'] = {} + resource['ResourceWords'] = resource_text.strip(" ").split(" ") + resource['Locations'], resource['Resources'] = dict(), {} + resource['Resources'] = {"resources": resource['ResourceWords']} + for each in locations: + resource['Locations'][each[0]] = {"long": float(each[1][0][1]), "lat": float(each[1][0][0])} + resource['Classification'] = int(get_classification(line, resource['ResourceWords'])) + print('Returning', resource) + return flask.jsonify(resource) + +# add routes for nodejs backend via here as well +@app.route('/', methods=['GET', 'OPTIONS']) +@cross_origin() +def base(): + with open('index.html', 'r') as f: + txt = f.readlines() + return ''.join(txt) + +if __name__ == '__main__': + port = int(os.environ.get('PORT', 5000)) + app.run(host='0.0.0.0', port=port, debug=True) diff --git a/classify_tweets_covid_infer.py b/classify_tweets_covid_infer.py new file mode 100644 index 0000000..2b8dd54 --- /dev/null +++ b/classify_tweets_covid_infer.py @@ -0,0 +1,419 @@ +import torch +from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler +# from keras.preprocessing.sequence import pad_sequences +from sklearn.model_selection import train_test_split +# from pytorch_pretrained_bert import BertTokenizer, BertConfig, BertModel +# from pytorch_pretrained_bert import BertAdam, BertForSequenceClassification +from tqdm import tqdm, trange +import pandas as pd +import io +import numpy as np +import matplotlib.pyplot as plt +import random +import pickle +import os +import torch.nn.functional as F +from torch import nn + +from types import SimpleNamespace +import pudb +import sys +from transformers import BertModel, BertConfig +from transformers import AutoTokenizer +# from bert_serving.client import BertClient +# bc = BertClient() + +try: + dataset = sys.argv[1] + if dataset not in ['nepal','italy']: + dataset='nepal' +except Exception as e: + dataset='nepal' + +# with open('DATA_2/INPUT/nepal_dict.p','rb') as handle: +# nepal_dict= pickle.load( handle) + +# with open('DATA_2/INPUT/italy_dict.p','rb') as handle: +# italy_dict= pickle.load(handle) + + +# nepal_dict ={} + +# if dataset =='nepal': +# need_file = open('DATA_2/INPUT/nepal_needs.txt', encoding="utf-8") +# offer_file = open('DATA_2/INPUT/nepal_offers.txt', encoding="utf-8") +# all_file =open('DATA_2/INPUT/nepal-all.txt', encoding="utf-8") +# else: +# need_file = open('./DATA_2/INPUT/italy_needs.txt', encoding="utf-8") +# offer_file = open('./DATA_2/INPUT/italy_offers.txt', encoding="utf-8") +# all_file =open('./DATA_2/INPUT/italy-all.txt', encoding="utf-8") + +# while(True): +# line = need_file.readline() +# if not line: break +# line=line.strip().split('<||>') +# nepal_dict[line[0]]=(line[1].lower(), 1) + +# while(True): +# line = offer_file.readline() +# if not line: break +# line=line.strip().split('<||>') +# nepal_dict[line[0]]=(line[1].lower(), 2) + +# while(True): +# line = all_file.readline() +# if not line: break +# line= line.strip().split('<||>') +# if line[0] not in nepal_dict: +# nepal_dict[line[0]]= (line[1].lower(),0) + +# print(len(nepal_dict)) + +# def create_train_test_data(nepal_dict): +# X=[[],[],[]] + +# for elem in nepal_dict: +# X[nepal_dict[elem][1]].append(nepal_dict[elem][0]) + +# random.shuffle(X[0]) +# random.shuffle(X[1]) +# random.shuffle(X[2]) + +# train = [(X[i][k],i) for i in range(0,3) for k in range(0,int(0.7*len(X[i]))) ] +# val = [(X[i][k],i) for i in range(0,3) for k in range(int(0.7*len(X[i])),int(0.8*len(X[i])))] +# test = [(X[i][k],i) for i in range(0,3) for k in range(int(0.8*len(X[i])), len(X[i]))] + +# random.shuffle(train) +# random.shuffle(val) +# random.shuffle(test) + +# return train, val, test + +# if dataset=='nepal': +# train_nepal, val_nepal, test_nepal = create_train_test_data(nepal_dict) +# else: +# train_nepal, val_nepal, test_nepal = create_train_test_data(italy_dict) + +# Reduce size of data +# train_nepal = train_nepal[:10] +# val_nepal = val_nepal[:10] +# test_nepal = test_nepal[:10] + +# train_nepal_sentences = ["[CLS] "+ text[0]+ " [SEP]" for text in train_nepal] +# val_nepal_sentences = ["[CLS] "+ text[0]+ " [SEP]" for text in val_nepal] +# test_nepal_sentences = ["[CLS] "+ text+ " [SEP]" for text in SENTENCES] +# train_italy_sentences = ["[CLS] "+ text[0]+ " [SEP]" for text in train_italy] +# val_italy_sentences = ["[CLS] "+ text[0]+ " [SEP]" for text in val_italy] +# test_italy_sentences = ["[CLS] "+ text[0]+ " [SEP]" for text in test_italy] + +# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True) + +# tokenized_nepal_train = [tokenizer.tokenize(sent) for sent in train_nepal_sentences] +# tokenized_nepal_val = [tokenizer.tokenize(sent) for sent in val_nepal_sentences] +# tokenized_nepal_test = [tokenizer.tokenize(sent) for sent in test_nepal_sentences] + +# tokenized_italy_train = [tokenizer.tokenize(sent) for sent in train_italy_sentences] +# tokenized_italy_val = [tokenizer.tokenize(sent) for sent in val_nepal_sentences] +# tokenized_italy_test = [tokenizer.tokenize(sent) for sent in test_italy_sentences] + +# train_nepal_labels = [elem[1] for elem in train_nepal] +# val_nepal_labels = [elem[1] for elem in val_nepal] +# test_nepal_labels = [elem[1] for elem in test_nepal] +# train_italy_labels = [elem[1] for elem in train_italy] +# val_italy_labels = [elem[1] for elem in val_italy] +# test_italy_labels = [elem[1] for elem in test_italy] + +# from transformers import AutoTokenizer +# tokenizer = AutoTokenizer.from_pretrained('bert-base-cased', use_fast=True) + +# MAX_LEN = 64 +# # pu.db +# # train_nepal_ids = tokenizer(train_nepal_sentences, padding="max_length", truncation=True, max_length=MAX_LEN)["input_ids"] +# # val_nepal_ids = tokenizer(val_nepal_sentences, padding="max_length", truncation=True, max_length=MAX_LEN)["input_ids"] +# test_nepal_ids = tokenizer(test_nepal_sentences, padding="max_length", truncation=True, max_length=MAX_LEN)["input_ids"] + + +# train_nepal_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_nepal_train], maxlen=MAX_LEN, dtype="long", truncating="post", padding="post") +# val_nepal_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_nepal_val], maxlen=MAX_LEN, dtype="long", truncating="post", padding="post") +# test_nepal_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_nepal_test], maxlen=MAX_LEN, dtype="long", truncating="post", padding="post") + +# train_nepal_masks = [] +# val_nepal_masks = [] +# test_nepal_masks = [] + +# for seq in train_nepal_ids: +# seq_mask = [float(i>0) for i in seq] +# train_nepal_masks.append(seq_mask) + +# for seq in val_nepal_ids: +# seq_mask = [float(i>0) for i in seq] +# val_nepal_masks.append(seq_mask) + +# for seq in test_nepal_ids: +# seq_mask = [float(i>0) for i in seq] +# test_nepal_masks.append(seq_mask) + +# pu.db +# train_nepal_ids = torch.FloatTensor(train_nepal_ids) +# val_nepal_ids = torch.FloatTensor(val_nepal_ids) +# test_nepal_ids = torch.FloatTensor(test_nepal_ids) + +# # train_nepal_masks = torch.LongTensor(train_nepal_masks) +# # val_nepal_masks = torch.LongTensor(val_nepal_masks) +# test_nepal_masks = torch.LongTensor(test_nepal_masks) + +# # train_nepal_labels = torch.LongTensor(train_nepal_labels) +# # val_nepal_labels = torch.LongTensor(val_nepal_labels) +# # pu.db +# test_nepal_labels = [0 for x in range(len(test_nepal_ids))] +# test_nepal_labels = torch.LongTensor(test_nepal_labels) + +# # pu.db +# # train_nepal_data = TensorDataset(train_nepal_ids, train_nepal_masks, train_nepal_labels) +# # val_nepal_data = TensorDataset(val_nepal_ids, val_nepal_masks, val_nepal_labels) +# test_nepal_data = TensorDataset(test_nepal_ids, test_nepal_masks, test_nepal_labels) + + + +class BertSentClassifier(torch.nn.Module): + def __init__(self, config): + super(BertSentClassifier, self).__init__() + self.num_labels = config.num_labels + self.bert = BertModel.from_pretrained(config.model_name) + for param in self.bert.base_model.parameters(): + param.requires_grad = False + self.dropout = torch.nn.Dropout(config.hidden_dropout_prob) + self.classifier = torch.nn.Linear(config.hidden_size, config.num_labels) + def forward(self, input_ids, token_type_ids =None, attention_mask= None): + # pu.db + output_here = self.bert(input_ids.long(), token_type_ids, attention_mask).last_hidden_state + # output_here = self.dropout(output_here) + logits = self.classifier(output_here[:,0,:]) + return F.log_softmax(logits, dim=1) + +class WrappedModel(nn.Module): + def __init__(self, module): + super(WrappedModel, self).__init__() + self.module = module # that I actually define. + def forward(self, x): + return self.module(x) + +config = {'hidden_dropout_prob':0.3, 'num_labels':3,'model_name':'bert-base-uncased', 'hidden_size':768, 'data_dir':'saved_models/',} +config = SimpleNamespace(**config) + +# model = BertModel.from_pretrained('bert-base-uncased') +# sent_bert = BertModel.from_pretrained(config.model_name) +model= BertSentClassifier(config) +# print("Loading Done") + +# import os +# # os.environ['CUDA_VISIBLE_DEVICES'] = '2' +# # model + +# param_optimizer = list(model.named_parameters()) +# no_decay = ['bias', 'gamma', 'beta'] +# optimizer_grouped_parameters = [ +# {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], +# 'weight_decay_rate': 0.01}, +# {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], +# 'weight_decay_rate': 0.0} +# ] + +# optimizer = torch.optim.Adam(optimizer_grouped_parameters, lr=2e-5) + + +# from sklearn.metrics import classification_report, f1_score + +# epochs = 150 + +# BATCH_SIZE = 1 + +# train_nepal_dataloader = DataLoader(train_nepal_data, shuffle = True, batch_size= BATCH_SIZE) +# val_nepal_dataloader = DataLoader(val_nepal_data, shuffle = False, batch_size= BATCH_SIZE) +# test_nepal_dataloader = DataLoader(test_nepal_data, shuffle = False, batch_size= BATCH_SIZE) + +# best_val=0 +# pu.db +# model_path = '{}/{}_covid.pth'.format(config.data_dir, dataset) +# model.cpu() +# for epoch in tqdm(range(epochs)): +# model.train() +# print(epoch) + +# tr_loss=0 +# batch_num=0 +# for step, batch in enumerate(train_nepal_dataloader): +# print("Done for batch = {}/{}".format(step,len(train_nepal_dataloader)), end='\r') +# b_ids, b_mask, b_labels = batch +# b_sent = train_nepal_sentences[(step * BATCH_SIZE) : (step * BATCH_SIZE) + BATCH_SIZE] +# # pu.db +# b_ids= b_ids.cuda() +# b_mask = b_mask.cuda() +# b_labels = b_labels.cuda() +# # weights = torch.Tensor([0.004461883549047657, 0.557096078912092, 0.4384420375388603]) +# # weights = torch.Tensor([0.01,0.55,0.44]) + +# optimizer.zero_grad() +# logits = model(b_ids, attention_mask=b_mask) +# # import pdb +# # pdb.set_trace() +# # pu.db +# loss = F.nll_loss(logits, b_labels.view(-1), reduction='sum') +# loss /= b_labels.view(-1).shape[0] +# loss.backward() +# optimizer.step() + +# tr_loss += loss.item() +# batch_num+=1 +# print("Train loss {}".format(tr_loss/batch_num)) +# # torch.save(model, model_path) + +# # pu.db +# # model.load_state_dict(torch.load(model_path)) + +# model.eval() + +# y_true=[] +# y_pred=[] + +# for step, batch in enumerate(val_nepal_dataloader): +# b_ids, b_mask, b_labels = batch +# b_ids= b_ids +# b_mask = b_mask +# with torch.no_grad(): +# output_here = sent_bert(b_ids.long(), None, b_mask).last_hidden_state +# logits = model(output_here, attention_mask=b_mask) +# logits = logits.detach().cpu().numpy() +# preds = np.argmax(logits, axis=1).flatten() +# b_labels = b_labels.flatten() +# y_true.extend(b_labels) +# y_pred.extend(preds) + + +# print(classification_report(y_true, y_pred)) +# f1= f1_score(y_true, y_pred, average='macro') +# if f1> best_val: +# best_val= f1 +# model_path = '{}/{}_bert_covid.pth'.format(config.data_dir, dataset) +# torch.save(model, model_path) +# print("Saved at val") + + +# model_path = '{}/{}_bert_service.pth'.format(config.data_dir, dataset) +# pu.db +# model = torch.load(model_path, map_location='cpu') +# model.cpu() +# model.eval() +# model = model + +# y_true=[] +# y_pred=[] +# for step, batch in enumerate(test_nepal_dataloader): +# b_ids, b_mask, b_labels = batch +# b_ids= b_ids +# b_mask = b_mask +# with torch.no_grad(): +# # pu.db +# logits = model.module(b_ids.cpu(), attention_mask=b_mask.cpu()) +# logits = logits.detach().cpu().numpy() +# preds = np.argmax(logits, axis=1).flatten() +# # b_labels = b_labels.flatten() +# # y_true.extend(b_labels) +# y_pred.extend(preds) + +model_path = '{}/{}_bert_covid.pth'.format(config.data_dir, dataset) +model = WrappedModel(model) +state_dict = torch.load(model_path,map_location='cpu') +# model = torch.load(model_path, map_location='cpu') +model.load_state_dict(state_dict) +model.cpu() +model.eval() +print("Done loading BERT model") + +# pu.db +# print(classification_report(y_true, y_pred)) +# print("\n") +# for i, sent in enumerate(SENTENCES): +# print(sent, end="") +# label = y_pred[i] +# if label == 0: +# print(": NEED") +# elif label == 1: +# print(": AVAIL") +# else: +# print(": OTHER") + +tokenizer = AutoTokenizer.from_pretrained('bert-base-cased', use_fast=True) +def evaluate_bert(text): + test_nepal_masks = [] + SENTENCES = text + test_nepal_sentences = ["[CLS] "+ text+ " [SEP]" for text in SENTENCES] + # test_nepal_here_sentences = ["[CLS] "+ text+ " [SEP]" for text in test_nepal_here] + MAX_LEN = 64 + test_nepal_ids = tokenizer(test_nepal_sentences, padding="max_length", truncation=True, max_length=MAX_LEN)["input_ids"] + for seq in test_nepal_ids: + seq_mask = [float(i>0) for i in seq] + test_nepal_masks.append(seq_mask) + test_nepal_ids = torch.FloatTensor(test_nepal_ids) + test_nepal_masks = torch.LongTensor(test_nepal_masks) + test_nepal_labels = [0 for x in range(len(test_nepal_ids))] + test_nepal_labels = torch.LongTensor(test_nepal_labels) + test_nepal_data = TensorDataset(test_nepal_ids, test_nepal_masks, test_nepal_labels) + BATCH_SIZE = 64 + test_nepal_dataloader = DataLoader(test_nepal_data, shuffle = False, batch_size= BATCH_SIZE) + y_true=[] + y_pred=[] + for step, batch in enumerate(test_nepal_dataloader): + b_ids, b_mask, b_labels = batch + b_ids= b_ids + b_mask = b_mask + with torch.no_grad(): + # pu.db + logits = model.module(b_ids.cpu(), attention_mask=b_mask.cpu()) + logits = logits.detach().cpu().numpy() + preds = np.argmax(logits, axis=1).flatten() + # b_labels = b_labels.flatten() + # y_true.extend(b_labels) + y_pred.extend(preds) + print(y_pred) + y_pred = [(a - 1) for a in y_pred] + # print("Classification: "+str(int(y_pred[0]) - 1)) + return y_pred + +if __name__ == "__main__": + text = input("Text ploxx: ") + # model = load_model() + evaluate_bert(text) + +''' +No weights, processed text. + +precision recall f1-score support + + 0 0.99 0.99 0.99 9641 + 1 0.73 0.65 0.68 99 + 2 0.75 0.69 0.72 265 + + accuracy 0.98 10005 + macro avg 0.82 0.78 0.80 10005 +weighted avg 0.98 0.98 0.98 10005 + + +No weights, un-processed text. + + + precision recall f1-score support + + 0 1.00 0.99 0.99 9641 + 1 0.61 0.75 0.67 97 + 2 0.67 0.84 0.74 267 + + accuracy 0.98 10005 + macro avg 0.76 0.86 0.80 10005 +weighted avg 0.98 0.98 0.98 10005 + + +Weights un-processed [0.01, 0.55, 0.44] + + +''' diff --git a/extract_resource_covid.py b/extract_resource_covid.py new file mode 100644 index 0000000..e55081d --- /dev/null +++ b/extract_resource_covid.py @@ -0,0 +1,62 @@ +text_org = input("Enter text, or press return for an example: ") +import location +import pudb +from nltk.tokenize import word_tokenize +if text_org == "": + text_org = "Oxygen producing unit at Princess Esra Hospital (Owaisi Group of Hospitals). #Oxygen #IndiaNeedsOxygen #IndiaFightsCOVID19 @aimim_national @imShaukatAli @asadowaisi @imAkbarOwaisi @warispathan @syedasimwaqar @Syed_Ruknuddin5 @ShahnawazAIMIM_ @Akhtaruliman5 https://t.co/vdZamB1wJl" +text = text_org.lower() + +places = location.return_location_list(text) +each_loc = [place[0] for place in places] +resources = { + "oxygen": "Oxygen", + "o2": "Oxygen", + "ventilator": "Ventilator", + "bed": "Beds", + "icu": "Beds", + "remdes": "Remdesivir", + "plasma": "Plasma", + "consultation": "Doctor", + "ambulance": "Ambulance" +} + +# pu.db +tokenized_text = word_tokenize(text) +print("\nOrig tokenized text:" + str(tokenized_text)) +for i in reversed(range(1, len(tokenized_text))): + # pu.db + word = tokenized_text[i] + word_prev = tokenized_text[i - 1] + if "#" in word_prev: + del tokenized_text[i] + +print("\nNew tokenized text:" + str(tokenized_text)) +text = "" +for word in tokenized_text: + text = text+word+" " + +places_to_remove = [] +resource_text = "" +for resource in resources: + if resource in each_loc: + places_to_remove.append(each_loc.index(resource)) + if resource in text: + resource_text = resource_text+resources[resource]+" " + +places_to_remove.sort(reverse=True) +for ptr in places_to_remove: + del places[ptr] + +resource_text = word_tokenize(resource_text) +resource_text = [w.lower() for w in resource_text] +resource_text = list(set(resource_text)) +print("\n\n\nText: "+str(text_org)) +print("\nLocation: "+str(places)) +print("\nResources: "+str(resource_text)) + +if "availab" in text: + print("\nType: Availability") +elif "need" in text or "require" in text: + print("\nType: Need") +else: + print("\nType: Other") diff --git a/location.py b/location.py index c3cd9e9..9f13edf 100644 --- a/location.py +++ b/location.py @@ -25,9 +25,10 @@ import random import wordsegment import jellyfish -from para_sentence import split_into_sentences +# from para_sentence import split_into_sentences import networkx as nx import geocoder +import pudb ps_stemmer=porter.PorterStemmer() @@ -46,7 +47,7 @@ stop_words_2=['i','me','we','us','you','u','she','her','his','he','him','it','they','them','who','which','whom','whose','that','this','these','those','anyone','someone','some','all','most','himself','herself','myself','itself','hers','ours','yours','theirs','to','in','at','for','from','etc',' ',','] stop_words.extend(stop_words_2) -stop_words.extend(['with', 'at', 'from', 'into', 'during', 'including', 'until', 'against', 'among', 'throughout', 'despite', 'towards', 'upon', 'concerning', 'of', 'to', 'in', 'for', 'on', 'by', 'about', 'like', 'through', 'over', 'before', 'between', 'after', 'since', 'without', 'under', 'within', 'along', 'following', 'across', 'behind', 'beyond', 'plus', 'except', 'but', 'up', 'out', 'around', 'down', 'off', 'above', 'near', 'and', 'or', 'but', 'nor', 'so', 'for', 'yet', 'after', 'although', 'as', 'as', 'if', 'long', 'because', 'before', 'even', 'if', 'even though', 'once', 'since', 'so', 'that', 'though', 'till', 'unless', 'until', 'what', 'when', 'whenever', 'wherever', 'whether', 'while', 'the', 'a', 'an', 'this', 'that', 'these', 'those', 'my', 'yours', 'his', 'her', 'its', 'ours', 'their', 'few', 'many', 'little', 'much', 'many', 'lot', 'most', 'some', 'any', 'enough', 'all', 'both', 'half', 'either', 'neither', 'each', 'every', 'other', 'another', 'such', 'what', 'rather', 'quite']) +stop_words.extend(['with', 'at', 'from', 'into', 'during', 'including', 'until', 'against', 'among', 'throughout', 'despite', 'towards', 'upon', 'concerning', 'of', 'to', 'in', 'for', 'on', 'by', 'about', 'like', 'through', 'over', 'before', 'between', 'after', 'since', 'without', 'under', 'within', 'along', 'following', 'across', 'behind', 'beyond', 'plus', 'except', 'but', 'up', 'out', 'around', 'down', 'off', 'above', 'near', 'and', 'or', 'but', 'nor', 'so', 'for', 'yet', 'after', 'although', 'as', 'as', 'if', 'long', 'because', 'before', 'even', 'if', 'even though', 'once', 'since', 'so', 'that', 'though', 'till', 'unless', 'until', 'what', 'when', 'whenever', 'wherever', 'whether', 'while', 'the', 'a', 'an', 'this', 'that', 'these', 'those', 'my', 'yours', 'his', 'her', 'its', 'ours', 'their', 'few', 'many', 'little', 'much', 'many', 'lot', 'most', 'some', 'any', 'enough', 'all', 'both', 'half', 'either', 'neither', 'each', 'every', 'other', 'another', 'such', 'what', 'rather', 'quite', 'oxygen', 'ventilator', 'bed', 'remdesivir', 'consultation', 'plasma', 'vir', 'se']) stop_words=list(set(stop_words)) stopword_file=open("DATA/Process_resources/stopword.txt",'r') stop_words.extend([line.rstrip() for line in stopword_file]) @@ -458,7 +459,7 @@ def NP_chunk(doc,text): return dep_places -with open('DATA/NP/NP_loc.p','rb') as handle: +with open('DATA/NP/IN_loc.p','rb') as handle: curr_loc_dict=pickle.load(handle) # false_names=false_names-set([i for i in curr_loc_dict]) @@ -476,6 +477,7 @@ def NP_chunk(doc,text): starttime=time.time() def return_location_list(text): + # pu.db lat_long=[] try: # print('\n') @@ -544,7 +546,7 @@ def return_location_list(text): if i =='' or i in false_names or ps_stemmer.stem(i) in false_names: continue if i.endswith('hospital') and len(i.split())>=3: - g=geocoder.osm(i+', Nepal') + g=geocoder.osm(i+', India') # print(g) if g.json!=None: lat_long.append((i,(g.json['lat'],g.json['lng']))) diff --git a/requirements.txt b/requirements.txt index ade5f8a..02bb73f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,8 @@ emoji==0.5.1 Flask-Cors==3.0.7 word2number==1.1 gunicorn==19.9.0 -https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz \ No newline at end of file +https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz +torch==1.8.0 +torchvision==0.9.0 +pudb==2019.1 +transformers==4.5.1 \ No newline at end of file