Drop MitreCVE table via migration

bedrock/security/management/commands/update_security_advisories.py

@@ -14,15 +14,14 @@
 from dateutil.parser import parse as parsedate
 
 from bedrock.base.sanitization import sanitize_html
-from bedrock.security.models import HallOfFamer, MitreCVE, Product, SecurityAdvisory
+from bedrock.security.models import HallOfFamer, Product, SecurityAdvisory
 from bedrock.security.utils import (
     FILENAME_RE,
     check_hof_data,
     mfsa_id_from_filename,
     parse_md_file,
     parse_yml_file,
     parse_yml_file_base,
-    update_advisory_bugs,
 )
 from bedrock.utils.git import GitRepo
 from bedrock.utils.management.cron_command import CronCommand
@@ -177,50 +176,6 @@ def add_hofers(filename, data):
         )
 
 
-def parse_cve_id(cve_id):
-    cve_year, cve_order = cve_id.split("-")[1:]
-    return int(cve_year), int(cve_order)
-
-
-def add_or_update_cve(data):
-    for cve_id, advisory in data["advisories"].items():
-        if not cve_id.startswith("CVE-"):
-            # skip advisories that are not CVE
-            continue
-
-        if not advisory.get("feed", True):
-            # skip advisories with `feed: false`
-            continue
-
-        cve_year, cve_order = parse_cve_id(cve_id)
-        update_advisory_bugs(advisory)
-        cve_title = advisory.get("cve_problemtype", advisory.get("title")) or ""
-        cve_data = {
-            "id": cve_id,
-            "year": cve_year,
-            "order": cve_order,
-            "title": cve_title,
-            "impact": advisory["impact"] or "",
-            "reporter": advisory["reporter"] or "",
-            "description": advisory.get("description", "") or "",
-            "bugs": advisory["bugs"],
-        }
-        try:
-            cve = MitreCVE.objects.get(id=cve_id)
-        except MitreCVE.DoesNotExist:
-            cve = MitreCVE(**cve_data)
-            cve.products = data["fixed_in"]
-            cve.mfsa_ids.append(data["mfsa_id"])
-        else:
-            cve.products = list(set(cve.products).union(data["fixed_in"]))
-            cve.mfsa_ids = list(set(cve.mfsa_ids).union([data["mfsa_id"]]))
-            for prop, value in cve_data.items():
-                if value:
-                    setattr(cve, prop, value)
-
-        cve.save()
-
-
 def update_db_from_file(filename):
     """
     Parse file for YAML and Markdown and update database.
@@ -240,8 +195,6 @@ def update_db_from_file(filename):
 
     data, html = parser(filename)
     html = sanitize_advisory_html(html)
-    if "advisories" in data:
-        add_or_update_cve(data)
     return add_or_update_advisory(data, html)
 
 
@@ -325,7 +278,6 @@ def printout(msg, ending=None):
             printout("Clearing all security advisories.")
             SecurityAdvisory.objects.all().delete()
             Product.objects.all().delete()
-            MitreCVE.objects.all().delete()
 
         if not no_git:
             printout("Updating repository.")

bedrock/security/migrations/0009_delete_mitrecve.py

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("security", "0008_alter_mitrecve_reporter_alter_product_name_and_more"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="MitreCVE",
+        ),
+    ]

bedrock/security/models.py

@@ -134,127 +134,3 @@ def year_quarter(self):
     def quarter_string(self):
         year, quarter = self.year_quarter
         return f"{self.ORDINALS[quarter]} Quarter {year}"
-
-
-class MitreCVE(models.Model):
-    id = models.CharField(max_length=15, primary_key=True, db_index=True)
-    year = models.IntegerField()
-    order = models.IntegerField()
-    title = models.CharField(max_length=200, blank=True)
-    impact = models.CharField(max_length=100, blank=True)
-    reporter = models.CharField(max_length=500, blank=True)
-    description = models.TextField()
-    products = JSONField(default="[]")
-    mfsa_ids = JSONField(default="[]")
-    bugs = JSONField(default="[]")
-
-    class Meta:
-        ordering = ("-year", "-order")
-
-    def __str__(self):
-        return self.id
-
-    def product_versions(self):
-        """Return a list of version numbers per product"""
-        prod_vers = {}
-        for prod in self.products:
-            prod_name, version = prod.rsplit(None, 1)
-            if prod_name in prod_vers:
-                prod_vers[prod_name].append(version)
-            else:
-                prod_vers[prod_name] = [version]
-
-        return prod_vers
-
-    def get_description(self):
-        versions = []
-        for prod_name, prod_versions in self.product_versions().items():
-            versions.extend(f"{prod_name} < {v}" for v in prod_versions)
-
-        description = self.description.strip()
-        if versions:
-            if len(versions) == 1:
-                vers_str = versions[0]
-            elif len(versions) == 2:
-                vers_str = " and ".join(versions)
-            else:
-                vers_str = ", ".join(versions[:-1]) + ", and " + versions[-1]
-
-            if description:
-                if description.endswith("."):
-                    description += " "
-                else:
-                    description += ". "
-
-            description += f"This vulnerability affects {vers_str}."
-
-        return description
-
-    def get_product_data(self):
-        product_data = []
-        for prod_name, versions in self.product_versions().items():
-            product_data.append(
-                {
-                    "product_name": prod_name,
-                    "version": {
-                        "version_data": [{"version_value": vers, "version_affected": "<"} for vers in versions],
-                    },
-                }
-            )
-
-        return product_data
-
-    def get_reference_data(self):
-        reference_data = [{"url": f"https://www.mozilla.org/security/advisories/mfsa{mfsa_id}/"} for mfsa_id in set(self.mfsa_ids)]
-        reference_data.extend([{"url": bug["url"]} for bug in self.bugs])
-        return reference_data
-
-    def feed_entry(self):
-        """Return a MITRE format data structure for the CVE
-
-        See https://github.com/CVEProject/automation-working-group/blob/master/cve_json_schema/DRAFT-JSON-file-format-v4.md
-        """
-        return {
-            "data_type": "CVE",
-            "data_format": "MITRE",
-            "data_version": "4.0",
-            "CVE_data_meta": {
-                "ID": self.id,
-                "ASSIGNER": "security@mozilla.org",
-            },
-            "affects": {
-                "vendor": {
-                    "vendor_data": [
-                        {
-                            "vendor_name": "Mozilla",
-                            "product": {
-                                "product_data": self.get_product_data(),
-                            },
-                        }
-                    ]
-                }
-            },
-            "problemtype": {
-                "problemtype_data": [
-                    {
-                        "description": [
-                            {
-                                "lang": "eng",
-                                "value": self.title,
-                            }
-                        ]
-                    }
-                ]
-            },
-            "references": {
-                "reference_data": self.get_reference_data(),
-            },
-            "description": {
-                "description_data": [
-                    {
-                        "lang": "eng",
-                        "value": self.get_description(),
-                    }
-                ]
-            },
-        }

bedrock/security/tests/test_models.py

@@ -5,7 +5,7 @@
 from datetime import date
 
 from bedrock.mozorg.tests import TestCase
-from bedrock.security.models import HallOfFamer, MitreCVE, Product
+from bedrock.security.models import HallOfFamer, Product
 
 
 class TestProduct(TestCase):
@@ -41,47 +41,3 @@ def test_year_quarter(self):
         assert hf3.quarter_string == "3rd Quarter 2016"
         assert hf4.year_quarter == (2015, 4)
         assert hf4.quarter_string == "4th Quarter 2015"
-
-
-class TestMitreCVE(TestCase):
-    cve_id_order = 100
-
-    def _create_cve(self, products=None):
-        self.cve_id_order += 1
-        products = products or ["Firefox 60", "Firefox 60.0.1"]
-        return MitreCVE.objects.create(
-            id=f"CVE-2018-{self.cve_id_order}",
-            year=2018,
-            order=self.cve_id_order,
-            title="A Testing Problem",
-            description="There was a problem.",
-            products=products,
-            mfsa_ids=["2018-11"],
-            bugs=[{"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1234567", "desc": "Bug 1234567"}],
-        )
-
-    def test_product_versions(self):
-        cve = self._create_cve()
-        assert cve.product_versions() == {"Firefox": ["60", "60.0.1"]}
-        cve = self._create_cve(["Firefox ESR 60.0.1", "Firefox ESR 52.8.0", "Firefox 60.0.1"])
-        assert cve.product_versions() == {"Firefox": ["60.0.1"], "Firefox ESR": ["60.0.1", "52.8.0"]}
-
-    def test_get_reference_data(self):
-        cve = self._create_cve()
-        assert cve.get_reference_data() == [
-            {"url": "https://www.mozilla.org/security/advisories/mfsa2018-11/"},
-            {"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1234567"},
-        ]
-
-    def test_get_description(self):
-        cve = self._create_cve()
-        assert cve.get_description() == "There was a problem. This vulnerability affects Firefox < 60 and Firefox < 60.0.1."
-        cve = self._create_cve(["Firefox ESR 60.0.1", "Firefox ESR 52.8.0", "Firefox 60.0.1"])
-        assert cve.get_description() == (
-            "There was a problem. This vulnerability affects Firefox ESR < 60.0.1, Firefox ESR < 52.8.0, and Firefox < 60.0.1."
-        )
-        cve = self._create_cve(["Firefox ESR 60.0.1"])
-        cve.description = "No punctuation"
-        assert cve.get_description() == "No punctuation. This vulnerability affects Firefox ESR < 60.0.1."
-        cve.description = "Punctuation but extra space. \n"
-        assert cve.get_description() == "Punctuation but extra space. This vulnerability affects Firefox ESR < 60.0.1."

bedrock/security/urls.py

@@ -28,8 +28,6 @@
     path("bug-bounty/web-hall-of-fame/", HallOfFameView.as_view(program="web"), name="security.bug-bounty.web-hall-of-fame"),
     path("advisories/", AdvisoriesView.as_view(), name="security.advisories"),
     re_path(r"^advisories/mfsa(?P<pk>\d{4}-\d{2,3})/$", AdvisoryView.as_view(), name="security.advisory"),
-    # FIXME: remove via issue 16519
-    # re_path(r"^advisories/cve-feed\.json$", mitre_cve_feed, name="security.advisories.cve_feed"),
     page("known-vulnerabilities/", "security/known-vulnerabilities.html"),
     page("known-vulnerabilities/older-vulnerabilities/", "security/older-vulnerabilities.html"),
     re_path(r"^known-vulnerabilities/(?P<slug>[a-z-]+)/$", ProductView.as_view(), name="security.product-advisories"),

bedrock/security/views.py

@@ -6,25 +6,17 @@
 from django.db.models import Q
 from django.urls import NoReverseMatch
 from django.utils.decorators import method_decorator
-from django.views.decorators.http import require_safe
 from django.views.generic import DetailView, ListView, RedirectView
 
-from jsonview.decorators import json_view
 from product_details import product_details
 from product_details.version_compare import Version
 
 from bedrock.base.urlresolvers import reverse
 from bedrock.mozorg.decorators import cache_control_expires
-from bedrock.security.models import HallOfFamer, MitreCVE, Product, SecurityAdvisory
+from bedrock.security.models import HallOfFamer, Product, SecurityAdvisory
 from lib.l10n_utils import LangFilesMixin, RequireSafeMixin
 
 
-@json_view
-@require_safe
-def mitre_cve_feed(request):
-    return [cve.feed_entry() for cve in MitreCVE.objects.all()]
-
-
 def product_is_obsolete(prod_name, version):
     """
     Return true if the product major version is not latest.

bin/export-db-to-sqlite.sh

@@ -190,7 +190,6 @@ python manage.py dumpdata \
     security.Product \
     security.SecurityAdvisory \
     security.HallOfFamer \
-    security.MitreCVE \
     releasenotes.ProductRelease \
     contentcards.ContentCard \
     utils.GitRepoState \