diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fa579daa..d6fd85a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -103,6 +103,7 @@ jobs: * ) scripts="bin";; esac . "${VENV_DIR}/${scripts}/activate" + python -m pip install --upgrade pip python -m pip install --default-timeout=1000 -r requirements.txt - name: Run tests shell: bash diff --git a/client/protocols/analytics.py b/client/protocols/analytics.py index 2d4e688a..4c1ae321 100644 --- a/client/protocols/analytics.py +++ b/client/protocols/analytics.py @@ -147,9 +147,7 @@ def is_correct(grading_results): provides the count of tests passed, failed or locked for a single question. Return True if all tests have passed. """ - if grading_results['locked'] > 0: - return False - return sum(grading_results.values()) == grading_results['passed'] + return grading_results['failed'] == 0 and grading_results['locked'] == 0 def first_failed_test(tests, scores): test_names = [t.name for t in tests] diff --git a/client/protocols/hinting.py b/client/protocols/hinting.py index 2b1f24bf..cf014330 100644 --- a/client/protocols/hinting.py +++ b/client/protocols/hinting.py @@ -4,21 +4,18 @@ obtains them from the hint generation server. Free response questions may be posed before and after hints are provided. """ -from client.sources.common import core -from client.sources.common import models as sources_models -from client.protocols.common import models as protocol_models -from client.utils import auth -from client.utils import format -from client.utils import prompt - import logging import random + import requests +from client.protocols.common import models as protocol_models +from client.utils import format, prompt from client.utils.printer import print_error log = logging.getLogger(__name__) + ##################### # Hinting Mechanism # ##################### @@ -31,11 +28,11 @@ class HintingProtocol(protocol_models.Protocol): HINT_ENDPOINT = 'api/hints' SMALL_EFFORT = 2 WAIT_ATTEMPTS = 5 - SUPPORTED_ASSIGNMENTS = ['cal/cs61a/fa17/hw09', 'cal/cs61a/fa17/hw10', - 'cal/cs61a/fa17/lab10'] + SUPPORTED_ASSIGNMENTS = ['ok/test/su16/ex', 'cal/cs61a/fa17/hw10', + 'cal/cs61a/fa17/lab10'] def run(self, messages): - """Determine if a student is elgible to recieve a hint. Based on their + """Determine if a student is eligible to recieve a hint. Based on their state, poses reflection questions. After more attempts, ask if students would like hints. If so, query @@ -74,38 +71,31 @@ def run(self, messages): continue stats = questions[question] is_solved = stats['solved'] == True - messages['hinting'][question] = {'prompts': {}, 'reflection': {}} - hint_info = messages['hinting'][question] + hint_info = messages['hinting']["question"] = {'name': question, 'prompts': {}, 'reflection': {}} - # Determine a users elgibility for a prompt + # Determine a users eligibility for a prompt # If the user just solved this question, provide a reflection prompt if is_solved: - hint_info['elgible'] = False + hint_info['eligible'] = False hint_info['disabled'] = 'solved' if self.args.hint: print("This question has already been solved.") continue elif stats['attempts'] < self.SMALL_EFFORT: - log.info("Question %s is not elgible: Attempts: %s, Solved: %s", + log.info("Question %s is not eligible: Attempts: %s, Solved: %s", question, stats['attempts'], is_solved) - hint_info['elgible'] = False + hint_info['eligible'] = False if self.args.hint: hint_info['disabled'] = 'attempt-count' print("You need to make a few more attempts before the hint system is enabled") continue else: # Only prompt every WAIT_ATTEMPTS attempts to avoid annoying user - if stats['attempts'] % self.WAIT_ATTEMPTS != 0: - hint_info['disabled'] = 'timer' - hint_info['elgible'] = False - log.info('Waiting for %d more attempts before prompting', - stats['attempts'] % self.WAIT_ATTEMPTS) - else: - hint_info['elgible'] = not is_solved + hint_info['eligible'] = not is_solved if not self.args.hint: - if hint_info['elgible']: + if hint_info['eligible']: with format.block("-"): print("To get hints, try using python3 ok --hint -q {}".format(question)) hint_info['suggested'] = True @@ -118,7 +108,7 @@ def run(self, messages): "... (This could take up to 30 seconds)")) pre_hint = random.choice(PRE_HINT_MESSAGES) print("In the meantime, consider: \n{}".format(pre_hint)) - hint_info['pre-prompt'] = pre_hint + hint_info['pre_hint'] = pre_hint log.info('Prompting for hint on %s', question) try: @@ -127,29 +117,25 @@ def run(self, messages): log.debug("Network error while fetching hint", exc_info=True) hint_info['fetch_error'] = True print_error("\r\nNetwork Error while generating hint. Try again later") - response = None continue - if response: - hint_info['response'] = response - - hint = response.get('message') - pre_prompt = response.get('pre-prompt') - post_prompt = response.get('post-prompt') - system_error = response.get('system-error') - log.info("Hint server response: {}".format(response)) - if not hint: - if system_error: - print("{}".format(system_error)) - else: - print("Sorry. No hints found for the current code. Try again making after some changes") - continue + hint_info['response'] = response + + hint = response.get('message') + post_prompt = response.get('post_prompt') + system_error = response.get('system_error') + log.info("Hint server response: {}".format(response)) + if not hint: + if system_error: + print("{}".format(system_error)) + else: + print("Sorry. No hints found for the current code. Try again making after some changes") + continue - # Provide padding for the the hint - print("\n{}".format(hint.rstrip())) + # Provide padding for the the hint + print("\n{}".format(hint.rstrip())) - if post_prompt: - results['prompts'][query] = prompt.explanation_msg(post_prompt) + prompt.explanation_msg(post_prompt) def query_server(self, messages, test): user = self.assignment.get_identifier() @@ -167,6 +153,7 @@ def query_server(self, messages, test): response.raise_for_status() return response.json() + SOLVE_SUCCESS_MSG = [ "If another student had the same error on this question, what advice would you give them?", "What did you learn from writing this program about things that you'll continue to do in the future?", @@ -175,16 +162,16 @@ def query_server(self, messages, test): ] ps_strategies_messages = ("Which of the following problem solving strategies will you attempt next?\n" -"- Manually running the code against the test cases\n" -"- Drawing out the environment diagram\n" -"- Try to solve the problem in another programming language and then translating\n" -"- Ensuring that all of the types in the input/output of the function match the specification\n" -"- Solve a few of the test cases manually and then trying to find a pattern\n" -"- Using print statements/inspecting the value of variables to debug") + "- Manually running the code against the test cases\n" + "- Drawing out the environment diagram\n" + "- Try to solve the problem in another programming language and then translating\n" + "- Ensuring that all of the types in the input/output of the function match the specification\n" + "- Solve a few of the test cases manually and then trying to find a pattern\n" + "- Using print statements/inspecting the value of variables to debug") PRE_HINT_MESSAGES = [ 'Could you describe what the function you are working is supposed to do at a high level?', - 'It would be helpful if you could explain the error to the computer:', # Rubber duck + 'It would be helpful if you could explain the error to the computer:', # Rubber duck 'Try to create a hypothesis for how that output was produced. This output is produced because ...', 'What is the simplest test that exposes this error?', ps_strategies_messages, @@ -192,7 +179,7 @@ def query_server(self, messages, test): 'What type of value (a string, a number etc) does the test indicate is outputted?', 'Are you convinced that the test case provided is correct?', 'Describe how exactly the program behaves incorrectly?', - 'In two sentences or less, explain how the error/output is produced by the code in the function', # Rubber Duck + 'In two sentences or less, explain how the error/output is produced by the code in the function', # Rubber Duck 'Are there lines that you suspect could be causing the program? Why those lines?', 'Have you tried to use print statements? On what line of the program would a print statement be useful?', 'Where is the last place you are sure your program was correct? How do you know?', @@ -202,5 +189,4 @@ def query_server(self, messages, test): ps_strategies_messages, ] - protocol = HintingProtocol diff --git a/client/protocols/unlock.py b/client/protocols/unlock.py index 12eafc6a..7aab6aef 100644 --- a/client/protocols/unlock.py +++ b/client/protocols/unlock.py @@ -41,7 +41,7 @@ def __init__(self, cmd_args, assignment): super().__init__(cmd_args, assignment) self.hash_key = assignment.name self.analytics = [] - self.guidance_util = guidance.Guidance("", assignment=assignment, suppress_warning_message=True) + self.guidance_util = guidance.Guidance() def run(self, messages): """Responsible for unlocking each test. @@ -69,9 +69,6 @@ def run(self, messages): log.info('Unlocking test {}'.format(test.name)) self.current_test = test.name - # Reset guidance explanation probability for every question - self.guidance_util.prompt_probability = guidance.DEFAULT_PROMPT_PROBABILITY - try: test.unlock(self.interact) except (KeyboardInterrupt, EOFError): @@ -154,16 +151,11 @@ def interact(self, unique_id, case_id, question_prompt, answer, choices=None, ra break else: correct = True - tg_id = -1 - misU_count_dict = {} rationale = "Unknown - Default Value" if not correct: - guidance_data = self.guidance_util.show_guidance_msg(unique_id, input_lines, - self.hash_key) - misU_count_dict, tg_id, printed_msg, rationale = guidance_data + printed_msg = self.guidance_util.show_guidance_msg(unique_id, input_lines) else: - rationale = self.guidance_util.prompt_with_prob() print("-- OK! --") printed_msg = ["-- OK! --"] @@ -175,10 +167,8 @@ def interact(self, unique_id, case_id, question_prompt, answer, choices=None, ra 'prompt': question_prompt, 'answer': input_lines, 'correct': correct, - 'treatment group id': tg_id, 'rationale': rationale, - 'misU count': misU_count_dict, - 'printed msg': printed_msg + 'printed msg': printed_msg, }) print() return input_lines diff --git a/client/sources/doctest/models.py b/client/sources/doctest/models.py index df372b69..e79288b4 100644 --- a/client/sources/doctest/models.py +++ b/client/sources/doctest/models.py @@ -101,7 +101,7 @@ def run(self, env): if success: return {'passed': 1, 'failed': 0, 'locked': 0} else: - return {'passed': 0, 'failed': 1, 'locked': 0} + return {'passed': 0, 'failed': 1, 'locked': 0, 'failed_outputs': [''.join(output_log)]} def score(self): format.print_line('-') @@ -133,4 +133,4 @@ def get_code(self): 'teardown': '', } } - return data \ No newline at end of file + return data diff --git a/client/sources/ok_test/concept.py b/client/sources/ok_test/concept.py index 6003390d..a7cfb486 100644 --- a/client/sources/ok_test/concept.py +++ b/client/sources/ok_test/concept.py @@ -37,8 +37,8 @@ def run(self, test_name, suite_number, env=None): results['locked'] += 1 continue - success = self._run_case(test_name, suite_number, - case, i + 1) + success, output = self._run_case(test_name, suite_number, + case, i + 1) assert success, 'Concept case should never fail while grading' results['passed'] += 1 return results diff --git a/client/sources/ok_test/doctest.py b/client/sources/ok_test/doctest.py index 8fdaf402..15fbe78f 100644 --- a/client/sources/ok_test/doctest.py +++ b/client/sources/ok_test/doctest.py @@ -48,6 +48,7 @@ def run(self, test_name, suite_number, env=None): results = { 'passed': 0, 'failed': 0, + 'failed_outputs': [], 'locked': 0, } @@ -66,7 +67,7 @@ def run(self, test_name, suite_number, env=None): results['locked'] += 1 continue - success = self._run_case(test_name, suite_number, + success, output = self._run_case(test_name, suite_number, case, i + 1) if not success and self.interactive: self.console.interact() @@ -75,6 +76,7 @@ def run(self, test_name, suite_number, env=None): results['passed'] += 1 else: results['failed'] += 1 + results['failed_outputs'].append(output) if not success and not self.verbose: # Stop at the first failed test diff --git a/client/sources/ok_test/models.py b/client/sources/ok_test/models.py index 2e294c3a..53b4ba56 100644 --- a/client/sources/ok_test/models.py +++ b/client/sources/ok_test/models.py @@ -60,7 +60,7 @@ def run(self, env): 'locked': int, } """ - passed, failed, locked = 0, 0, 0 + passed, failed, locked, failed_outputs = 0, 0, 0, [] for i, suite in enumerate(self.suites): if self.run_only and self.run_only != i + 1: continue @@ -72,6 +72,7 @@ def run(self, env): passed += results['passed'] failed += results['failed'] locked += results['locked'] + failed_outputs += results.get('failed_outputs', []) if not self.verbose and (failed > 0 or locked > 0): # Stop at the first failed test @@ -90,6 +91,7 @@ def run(self, env): 'passed': passed, 'failed': failed, 'locked': locked, + 'failed_outputs': failed_outputs, } def score(self, env=None): @@ -308,12 +310,14 @@ def _run_case(self, test_name, suite_number, case, case_number): output_log = output.get_log(log_id) output.remove_log(log_id) + output_str = ''.join(output_log) + if not success or self.verbose: - print(''.join(output_log)) + print(output_str) if not success: short_name = self.test.get_short_name() # TODO: Change when in notebook mode print('Run only this test case with ' '"python3 ok -q {} --suite {} --case {}"'.format( short_name, suite_number, case_number)) - return success + return success, output_str diff --git a/client/sources/ok_test/wwpp.py b/client/sources/ok_test/wwpp.py index bacd21c6..4036e8fc 100644 --- a/client/sources/ok_test/wwpp.py +++ b/client/sources/ok_test/wwpp.py @@ -38,7 +38,7 @@ def run(self, test_name, suite_number, env=None): results['locked'] += 1 continue - success = self._run_case(test_name, suite_number, + success, output = self._run_case(test_name, suite_number, case, i + 1) assert success, 'Wwpp case should never fail while grading' results['passed'] += 1 diff --git a/client/utils/guidance.py b/client/utils/guidance.py index e5ceb0fe..5f076be8 100644 --- a/client/utils/guidance.py +++ b/client/utils/guidance.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from client.utils import assess_id_util from client.utils import prompt from client.utils import format @@ -19,388 +21,73 @@ the default "Not Quite Try Again" message will show some kind of message that will target that type of misunderstanding. -This utility object requires internet access to determine what treatment group they are assigned -to. The different treatment groups will have varying threshold level of answers as well as different -messages and other differences. It will contact the server defined below in the variable TGSERVER with -the user's email and lab assignment to get the treatment group number. - -Commonly used acronyms: -TG = treatment group number -KI = Type of targeted understanding -misU = Type of misunderstanding the student is showing -wa = wrong answer - -The LOCAL_TG_FILE will hold what treatment group number the student is part of. -The OK_GUIDANCE_FILE will facilitate the generation of guided messages. It will hold the necessary info -to know what type of misunderstanding an answer has as well as what guidance message to show. +This utility object requires internet access to fetch hints from the server. """ -TGSERVER = "https://tg-server.app.cs61a.org/" -TG_SERVER_ENDING = "/unlock_tg" +HINT_SERVER = "http://localhost:5000" +WWPD_HINTING_ENDPOINT = "/api/wwpd_hints" -LOCAL_TG_FILE = "tests/tg.ok_tg" -OK_GUIDANCE_FILE = "tests/.ok_guidance" GUIDANCE_DEFAULT_MSG = "-- Not quite. Try again! --" -EMPTY_MISUCOUNT_TGID_PRNTEDMSG = ({}, -1, [], "Unknown Rationale") -COUNT_FILE_PATH = "tests/misUcount.json" -TG_CONTROL = 0 -# If student forces guidance messages to show, we will assign treatment -# group number below -GUIDANCE_FLAG_TG_NUMBER = 1 -# If set_tg() fails, we will default to this treatment group number -TG_ERROR_VALUE = -1 - -# Question prompt for misunderstanding recognition -EXPLANTION_PROMPT = """ -To help CS 61A provide better hints to future students, please take -a moment to explain your answer. -""".strip() -CONFIRM_BLANK_EXPLANATION = """ -Are you sure you don't want to answer? Explaining your answer can -improve your understanding of the question. Press Enter again to skip -the explanation and continue unlocking. -""".strip() - -DEFAULT_PROMPT_PROBABILITY = 0.10 +GUIDANCE_NO_HINTS_MSG = "-- Sorry, there aren't any hints available. --" +GUIDANCE_HINT_LOADING_MSG = "-- Thinking of a hint... (This could take up to 30 seconds) --" -# These lambda functions allow us to map from a certain type of misunderstanding to -# the desired targeted guidance message we want to show. -# lambda for control or treatment group where we want nothing to happen -# Knowledge integration treatment group lambda that is answer specific -# lambda for returning an answer + misunderstanding specific message +HINT_THRESHOLD = 3 -lambda_string_key_to_func = { - 'none': lambda info, strMisU: None, - 'ki': lambda info, strMisU: info['ki'], - 'misU2Msg': lambda info, strMisU: info['dictMisU2Msg'].get(strMisU), - 'tag2KIMsg': lambda info, strMisU: info['dictTag2KIMsg'].get(strMisU), - 'tag2ConceptMsg': lambda info, strMisU: info['dictTag2ConceptMsg'].get(strMisU) -} class Guidance: - def __init__(self, current_working_dir, assignment=None, suppress_warning_message=False): - """ - Initializing everything we need to the default values. If we catch - an error when opening the JSON file, we flagged it as error. - """ - self.tg_id = -1 - self.prompt_probability = DEFAULT_PROMPT_PROBABILITY - self.assignment = assignment - if assignment: - self.assignment_name = assignment.name.replace(" ", "") - else: - self.assignment_name = "" - - self.current_working_dir = current_working_dir - try: - with open(current_working_dir + OK_GUIDANCE_FILE, "r") as f: - self.guidance_json = json.load(f) - self.load_error = False - if not self.validate_json(): - raise ValueError("JSON did not validate") - self.guidance_json = self.guidance_json['db'] - except (OSError, IOError, ValueError): - if not suppress_warning_message: - log.warning("Failed to read .ok_guidance file. It may not exist") - self.load_error = True - log.debug("Guidance loaded with status: %s", not self.load_error) + msg_cache = {} + fail_cnt = defaultdict(int) - def validate_json(self): - """ Ensure that the checksum matches. """ - if not hasattr(self, 'guidance_json'): - return False + @staticmethod + def lookup_key(unlock_id, selected_options): + return unlock_id, tuple(selected_options) - checksum = self.guidance_json.get('checksum') - contents = self.guidance_json.get('db') - - hash_key = ("{}{}".format(json.dumps(contents, sort_keys=True), - self.assignment.endpoint).encode()) - - digest = hashlib.md5(hash_key).hexdigest() - - if not checksum: - log.warning("Checksum on guidance not found. Invalidating file") - return False - if digest != checksum: - log.warning("Checksum %s did not match actual digest %s", checksum, digest) - return False - return True - - def get_aid_from_anum(self, num): - """ Return the unique id (str) from the assesment id number. """ - return self.guidance_json['dictAssessNum2AssessId'].get(num) - - def show_guidance_msg(self, unique_id, input_lines, hash_key, - guidance_flag=False): + def get_hints(self, unlock_id, selected_options): + key = self.lookup_key(unlock_id, selected_options) + if key not in self.msg_cache: + try: + response = requests.post(HINT_SERVER + WWPD_HINTING_ENDPOINT, json=dict( + unlock_id=unlock_id, + selected_options=selected_options, + ), timeout=35) + response.raise_for_status() + self.msg_cache[key] = response.json()["hints"] + except (requests.exceptions.RequestException, requests.exceptions.BaseHTTPError): + self.msg_cache[key] = [] + return self.msg_cache[key] + + def show_guidance_msg(self, unique_id, input_lines): """ Based on the student's answer (input_lines), we grab each associated message if its corresponding misunderstanding's count is above the threshold """ - if self.load_error: - print(GUIDANCE_DEFAULT_MSG) - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - - response = repr(input_lines) - self.set_tg() - log.info("Guidance TG is %d", self.tg_id) - - if self.tg_id == TG_ERROR_VALUE: - # If self.tg_id == -1, there was an error when trying to access the server - log.warning("Error when trying to access server. TG == -1") - print(GUIDANCE_DEFAULT_MSG) - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - - lambda_string_key = self.guidance_json[ - 'dictTg2Func'].get(str(self.tg_id)) - - if not lambda_string_key: - log.info("Cannot find the correct lambda in the dictionary.") - print(GUIDANCE_DEFAULT_MSG) - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - log.info("Lambda Group: %s", lambda_string_key) - - lambda_info_misu = lambda_string_key_to_func.get(lambda_string_key) - if not lambda_info_misu: - log.info("Cannot find info misU given the lambda string key.") - print(GUIDANCE_DEFAULT_MSG) - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - - shorten_unique_id = assess_id_util.canonicalize(unique_id) - # Try to get the info dictionary for this question. Maps wrong answer - # to dictionary - assess_dict_info = self.guidance_json[ - 'dictAssessId2Info'].get(shorten_unique_id) - if not assess_dict_info: - log.info("shorten_unique_id %s is not in dictAssessId2Info", repr(shorten_unique_id)) - print(GUIDANCE_DEFAULT_MSG) - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - - wa_details = assess_dict_info['dictWA2DictInfo'].get(response) - if not wa_details: - log.info("Cannot find the wrong answer in the WA2Dict for this assesment.") - lst_mis_u = [] - else: - lst_mis_u = wa_details.get('lstMisU', []) - - # No list of misunderstandings for this wrong answer, default message - if not lst_mis_u: - log.info("Cannot find the list of misunderstandings.") - - wa_count_threshold = self.guidance_json['wrongAnsThresh'] - wa_lst_assess_num = assess_dict_info['dictWA2LstAssessNum_WA'] - msg_id_set = set() - should_skip_propagation = self.tg_id == 3 or self.tg_id == 4 - - answerDict, countData = self.get_misUdata() - prev_responses = answerDict.get(shorten_unique_id, []) - - # Confirm that this WA has not been given before - seen_before = response in prev_responses - if seen_before: - log.info("Answer has been seen before: {}".format(response)) - else: - answerDict[shorten_unique_id] = prev_responses + [response] - self.save_misUdata(answerDict, countData) - # Lookup the list of assessNum and WA related to this wrong answer - # in the question's dictWA2LstAssessNum_WA - lst_assess_num = wa_lst_assess_num.get(response, []) - if not lst_assess_num: - log.info("Cannot get the lst of assess nums given this reponse.") - log.debug("Related LST_ASSESS_NUM: %s", lst_assess_num) + self.fail_cnt[unique_id] += 1 - # Check if the current wrong answer is in the question's dictWA2DictInfo - if wa_details: - log.info("The current wrong answer (%s) is in dictWA2DictInfo", response) - # Check in answerDict to see if the student has ever given - # any of these wrong answers (sourced from dictWA2LstAssessNum_WA) - num_prev_responses = 1 + fetching = False - for other_num, other_resp in lst_assess_num: - # Get assess_id - other_id = self.get_aid_from_anum(other_num) - log.info("Checking if %s is in answerDict[%s]", other_resp, repr(other_id)) - if other_resp in answerDict.get(other_id, []): - log.debug("%s is in answerDict[%s]", other_resp, repr(other_id)) - num_prev_responses += 1 + if self.fail_cnt[unique_id] >= HINT_THRESHOLD: + if self.lookup_key(unique_id, input_lines) not in self.msg_cache: + print(GUIDANCE_HINT_LOADING_MSG) + fetching = True - log.info("Has given %d previous responses in lst_assess_num", num_prev_responses) + hints = self.get_hints(unique_id, input_lines) - if not should_skip_propagation: - # Increment countDict by the number of wrong answers seen - # for each tag assoicated with this wrong answerDict - increment = num_prev_responses - for misu in lst_mis_u: - log.info("Updating the count of misu: %s by %s", misu, increment) - countData[misu] = countData.get(misu, 0) + increment + if not hints: + log.info("No messages to display.") + if fetching: + print(GUIDANCE_NO_HINTS_MSG) + print(GUIDANCE_DEFAULT_MSG) + return [] - for misu in lst_mis_u: - log.debug("Misu: %s has count %s", misu, countData.get(misu, 0)) - if countData.get(misu, 0) >= wa_count_threshold: - msg_info = lambda_info_misu(wa_details, misu) - if msg_info: - msg_id_set.add(msg_info) + print("\n-- Helpful Hint --") - elif not should_skip_propagation: - # Lookup the lst_mis_u of each wrong answer in the list of wrong - # answers related to the current wrong answer (lst_assess_num), - # using dictAssessNum2AssessId - assess_num_to_aid = self.guidance_json['dictAssessNum2AssessId'] - log.debug("Looking up the lst_misu_u of all related WA") - - # misu -> list of wrong answers for that - related_misu_tags_dict = {} - - for related_num, related_resp in lst_assess_num: - related_aid = assess_num_to_aid.get(related_num) - log.info("Getting related resp %s for AID %s", repr(related_aid), related_resp) - resp_seen_before = related_resp in answerDict.get(related_aid, []) - - if not resp_seen_before: - continue - - # Get the lst_misu for this asssigmment - related_info = self.guidance_json['dictAssessId2Info'].get(related_aid) - if not related_info: - log.info("Could not find related id: %s in info dict", - related_aid) - continue - related_wa_info = related_info['dictWA2DictInfo'].get(related_resp) - - if not related_info: - log.info("Could not find response %s in %s info dict", - related_resp, related_aid) - continue - - related_misu_list = related_wa_info.get('lstMisU', []) - log.info("The related MISU list is %s", related_misu_list) - - for misu in related_misu_list: - existing_resps = related_misu_tags_dict.get(misu, []) - # Add dictWA2DictInfo to list of responses for this misunderstanding. - related_misu_tags_dict[misu] = existing_resps + [related_wa_info] - # Increment countDict for each tag in the set of tags for each related resp - countData[misu] = countData.get(misu, 0) + 1 - - for misu, lst_wa_info in related_misu_tags_dict.items(): - if countData.get(misu, 0) >= wa_count_threshold: - for wa_info in lst_wa_info: - msg_id_set.add(lambda_info_misu(wa_info, misu)) - else: - log.info("misu %s seen %s/%s times", - misu, countData.get(misu, 0), wa_count_threshold) - - self.save_misUdata(answerDict, countData) - - wa_lst_explain_responses = assess_dict_info.get('lstWrongAnsWatch', []) - if response in wa_lst_explain_responses: - rationale = self.prompt_with_prob(orig_response=input_lines, prob=1.0) - else: - rationale = self.prompt_with_prob(orig_response=input_lines) - - if len(msg_id_set) == 0: - log.info("No messages to display.") - print(GUIDANCE_DEFAULT_MSG) - return (countData, self.tg_id, [], rationale) - - print("\n-- Helpful Hint --") - - printed_out_msgs = [] - for message_id in msg_id_set: - msg = self.guidance_json['dictId2Msg'].get(str(message_id)) - if msg: - printed_out_msgs.append(msg) - print(msg) + printed_out_msgs = [] + for hint in hints: + printed_out_msgs.append(hint) + print(hint) print("-"*18) - else: - log.info("{} did not have a message".format(message_id)) - print() - print(GUIDANCE_DEFAULT_MSG) - - return (countData, self.tg_id, printed_out_msgs, rationale) - - def get_misUdata(self): - # Creates a new folder inside tests that stores the number of misU per - # assignment - if os.path.isfile(self.current_working_dir + COUNT_FILE_PATH): - with open(self.current_working_dir + COUNT_FILE_PATH, 'r') as f: - jsonDic = json.load(f) - answerDict = jsonDic["answerDict"] - countData = jsonDic["countData"] + return printed_out_msgs else: - countData = {} - answerDict = {} - - return answerDict, countData - - def save_misUdata(self, answerDict, countData): - data = { - "countData": countData, - "answerDict": answerDict, - } - log.info("Attempting to save response/count dict") - with open(self.current_working_dir + COUNT_FILE_PATH, "w") as f: - json.dump(data, f) - return data - - def set_tg(self): - """ Try to grab the treatment group number for the student. - If there is no treatment group number available, request it - from the server. - """ - # Checks to see the student currently has a treatment group number. - if not os.path.isfile(self.current_working_dir + LOCAL_TG_FILE): - cur_email = self.assignment.get_student_email() - log.info("Current email is %s", cur_email) - if not cur_email: - self.tg_id = -1 - return EMPTY_MISUCOUNT_TGID_PRNTEDMSG - - tg_url = ("{}{}/{}{}" - .format(TGSERVER, cur_email, self.assignment_name, - TG_SERVER_ENDING)) - try: - log.info("Accessing treatment server at %s", tg_url) - data = requests.get(tg_url, timeout=1).json() - except IOError: - data = {"tg": -1} - log.warning("Failed to communicate to server", exc_info=True) - - if data.get("tg") is None: - log.warning("Server returned back a bad treatment group ID.") - data = {"tg": -1} - - with open(self.current_working_dir + LOCAL_TG_FILE, "w") as fd: - fd.write(str(data["tg"])) - - tg_file = open(self.current_working_dir + LOCAL_TG_FILE, 'r') - self.tg_id = int(tg_file.read()) - - def prompt_with_prob(self, orig_response=None, prob=None): - """Ask for rationale with a specific level of probability. """ - # Disable opt-out. - # if self.assignment.cmd_args.no_experiments: - # log.info("Skipping prompt due to --no-experiments") - # return "Skipped due to --no-experiments" - if self.load_error: - return 'Failed to read guidance config file' - if hasattr(self.assignment, 'is_test'): - log.info("Skipping prompt due to test mode") - return "Test response" - - if prob is None: - prob = self.prompt_probability - - if random.random() > prob: - log.info("Did not prompt for rationale: Insufficient Probability") - return "Did not prompt for rationale" - with format.block(style="-"): - rationale = prompt.explanation_msg(EXPLANTION_PROMPT, - short_msg=CONFIRM_BLANK_EXPLANATION) - - if prob is None: - # Reduce future prompt likelihood - self.prompt_probability = 0 - if orig_response: - print('Thanks! Your original response was: {}'.format('\n'.join(orig_response))) - - return rationale + print() + print(GUIDANCE_DEFAULT_MSG) + return [] diff --git a/tests/sources/doctest/models_test.py b/tests/sources/doctest/models_test.py index 32a31363..3e0213c8 100644 --- a/tests/sources/doctest/models_test.py +++ b/tests/sources/doctest/models_test.py @@ -164,11 +164,13 @@ def testRun_partialFail(self): >>> 2 + 2 5 """) + results = test.run(None) + self.assertEqual(len(results.pop('failed_outputs')), 1) self.assertEqual({ 'passed': 0, 'failed': 1, 'locked': 0, - }, test.run(None)) + }, results) def testRun_completeFail(self): test = self.makeDoctest(""" @@ -177,11 +179,13 @@ def testRun_completeFail(self): >>> 2 + 2 5 """) + results = test.run(None) + self.assertEqual(len(results.pop('failed_outputs')), 1) self.assertEqual({ 'passed': 0, 'failed': 1, 'locked': 0, - }, test.run(None)) + }, results) def testRun_solitaryPS1(self): test = self.makeDoctest(""" @@ -191,11 +195,13 @@ def testRun_solitaryPS1(self): >>> 1 1 """) + results = test.run(None) + self.assertEqual(len(results.pop('failed_outputs')), 1) self.assertEqual({ 'passed': 0, 'failed': 1, 'locked': 0, - }, test.run(None)) + }, results) def testRun_solitaryPS2(self): test = self.makeDoctest(""" diff --git a/tests/sources/ok_test/doctest_test.py b/tests/sources/ok_test/doctest_test.py index 8ffe1b07..5ab27675 100644 --- a/tests/sources/ok_test/doctest_test.py +++ b/tests/sources/ok_test/doctest_test.py @@ -20,11 +20,13 @@ def testConstructor_noCases(self): self.fail() def callsRun(self, test, passed, failed, locked): + test_results = test.run(self.TEST_NAME, self.SUITE_NUMBER, None) + self.assertEqual(len(test_results.pop('failed_outputs')), failed) self.assertEqual({ 'passed': passed, 'locked': locked, 'failed': failed, - }, test.run(self.TEST_NAME, self.SUITE_NUMBER, None)) + }, test_results) def testConstructor_basicCases(self): try: diff --git a/tests/sources/ok_test/models_test.py b/tests/sources/ok_test/models_test.py index f45069de..71f40c65 100644 --- a/tests/sources/ok_test/models_test.py +++ b/tests/sources/ok_test/models_test.py @@ -48,6 +48,7 @@ def callsRun(self, passed, failed, locked): 'passed': passed, 'failed': failed, 'locked': locked, + 'failed_outputs': [], }, test.run(None)) self.mockSuite1.return_value.run.assert_called_with(self.NAME, 1, None) self.mockSuite2.return_value.run.assert_called_with(self.NAME, 2, None) @@ -128,6 +129,7 @@ def testRun_noSuites(self): 'passed': 0, 'failed': 0, 'locked': 0, + 'failed_outputs': [], }, test.run(None)) def testRun_correctResults(self): diff --git a/tests/utils/guidance_test.py b/tests/utils/guidance_test.py index 3ca58476..089b176f 100644 --- a/tests/utils/guidance_test.py +++ b/tests/utils/guidance_test.py @@ -41,8 +41,7 @@ def setUp(self): self.cmd_args = mock.Mock() self.assignment = mock.Mock(endpoint="cal/cs61a/sp16/test") self.proto = unlock.protocol(self.cmd_args, self.assignment) - self.proto.guidance_util = guidance.Guidance(self.GUIDANCE_DIRECTORY, - self.assignment) + self.proto.guidance_util = guidance.Guidance() self.proto.guidance_util.set_tg = self.mockSet_TG self.proto.current_test = self.TEST self.proto._verify = self.mockVerify