diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 094a4f1c..1e5bb92c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,13 @@ Release notes - Add dark theme support in UI. https://github.com/nexB/dejacode/issues/25 +- Add "Load Packages from SBOMs", "Import scan results", and + "Pull ScanCode.io project data" feature as Product action in the REST API. + https://github.com/nexB/dejacode/issues/59 + +- Refactor the "Import manifest" feature as "Load SBOMs". + https://github.com/nexB/dejacode/issues/61 + ### Version 5.0.1 - Improve the stability of the "Check for new Package versions" feature. diff --git a/product_portfolio/api.py b/product_portfolio/api.py index 869e5a1d..c8c4a1d8 100644 --- a/product_portfolio/api.py +++ b/product_portfolio/api.py @@ -11,7 +11,10 @@ import django_filters from rest_framework import permissions from rest_framework import serializers +from rest_framework import status +from rest_framework.decorators import action from rest_framework.permissions import SAFE_METHODS +from rest_framework.response import Response from component_catalog.api import KeywordsField from component_catalog.api import PackageEmbeddedSerializer @@ -31,6 +34,9 @@ from dje.filters import NameVersionFilter from dje.permissions import assign_all_object_permissions from product_portfolio.filters import ComponentCompletenessAPIFilter +from product_portfolio.forms import ImportFromScanForm +from product_portfolio.forms import LoadSBOMsForm +from product_portfolio.forms import PullProjectDataForm from product_portfolio.models import CodebaseResource from product_portfolio.models import Product from product_portfolio.models import ProductComponent @@ -191,6 +197,57 @@ class Meta: ) +class LoadSBOMsFormSerializer(serializers.Serializer): + """Serializer equivalent of LoadSBOMsForm, used for API documentation.""" + + input_file = serializers.FileField( + required=True, + help_text=LoadSBOMsForm.base_fields["input_file"].label, + ) + update_existing_packages = serializers.BooleanField( + required=False, + default=False, + help_text=LoadSBOMsForm.base_fields["update_existing_packages"].help_text, + ) + scan_all_packages = serializers.BooleanField( + required=False, + default=False, + help_text=LoadSBOMsForm.base_fields["scan_all_packages"].help_text, + ) + + +class ImportFromScanSerializer(serializers.Serializer): + """Serializer equivalent of ImportFromScanForm, used for API documentation.""" + + upload_file = serializers.FileField( + required=True, + ) + create_codebase_resources = serializers.BooleanField( + required=False, + default=False, + help_text=ImportFromScanForm.base_fields["create_codebase_resources"].help_text, + ) + stop_on_error = serializers.BooleanField( + required=False, + default=False, + help_text=ImportFromScanForm.base_fields["stop_on_error"].help_text, + ) + + +class PullProjectDataSerializer(serializers.Serializer): + """Serializer equivalent of PullProjectDataForm, used for API documentation.""" + + project_name_or_uuid = serializers.CharField( + required=True, + help_text=PullProjectDataForm.base_fields["project_name_or_uuid"].label, + ) + update_existing_packages = serializers.BooleanField( + required=False, + default=False, + help_text=PullProjectDataForm.base_fields["update_existing_packages"].help_text, + ) + + class ProductViewSet(CreateRetrieveUpdateListViewSet): queryset = Product.objects.none() serializer_class = ProductSerializer @@ -240,6 +297,72 @@ def perform_create(self, serializer): super().perform_create(serializer) assign_all_object_permissions(self.request.user, serializer.instance) + @action(detail=True, methods=["post"], serializer_class=LoadSBOMsFormSerializer) + def load_sboms(self, request, *args, **kwargs): + """ + Load Packages from SBOMs. + + DejaCode supports the following SBOM formats: + * CycloneDX BOM as JSON bom.json and .cdx.json, + * SPDX document as JSON .spdx.json, + * AboutCode .ABOUT files, + + Multiple SBOMs: You can provide multiple SBOMs by packaging them into a zip + archive. DejaCode will handle and process them accordingly. + """ + product = self.get_object() + + form = LoadSBOMsForm(data=request.POST, files=request.FILES) + if not form.is_valid(): + return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) + + form.submit(product=product, user=request.user) + return Response({"status": "SBOM file submitted to ScanCode.io for inspection."}) + + @action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer) + def import_from_scan(self, request, *args, **kwargs): + """ + Import the scan results in the Product. + + Upload a ScanCode.io or ScanCode-toolkit JSON output file. + """ + product = self.get_object() + + form = ImportFromScanForm(user=request.user, data=request.POST, files=request.FILES) + if not form.is_valid(): + return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) + + try: + warnings, created_counts = form.save(product=product) + except ValidationError as error: + return Response(error.messages, status=status.HTTP_400_BAD_REQUEST) + + if not created_counts: + msg = "Nothing imported." + else: + msg = "Imported from Scan: " + msg += ", ".join([f"{value} {key}" for key, value in created_counts.items()]) + return Response({"status": msg}) + + @action(detail=True, methods=["post"], serializer_class=PullProjectDataSerializer) + def pull_scancodeio_project_data(self, request, *args, **kwargs): + """ + Pull data from a ScanCode.io Project to import all its Discovered Packages. + Imported Packages will be assigned to this Product. + """ + product = self.get_object() + + form = PullProjectDataForm(data=request.POST) + if not form.is_valid(): + return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) + + try: + form.submit(product=product, user=request.user) + except ValidationError as error: + return Response(error.messages, status=status.HTTP_400_BAD_REQUEST) + + return Response({"status": "Packages import from ScanCode.io in progress..."}) + class BaseProductRelationSerializer(ValidateLicenseExpressionMixin, DataspacedSerializer): product = NameVersionHyperlinkedRelatedField( diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index 5c5bb8b9..2b901568 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -7,6 +7,8 @@ # from django import forms +from django.core.exceptions import ValidationError +from django.db import transaction from django.forms import BaseModelFormSet from django.forms.formsets import DELETION_FIELD_NAME from django.urls import reverse_lazy @@ -28,6 +30,8 @@ from component_catalog.license_expression_dje import LicenseExpressionFormMixin from component_catalog.models import Component from component_catalog.programming_languages import PROGRAMMING_LANGUAGES +from dejacode_toolkit.scancodeio import ScanCodeIO +from dje import tasks from dje.fields import SmartFileField from dje.forms import ColorCodeFormMixin from dje.forms import DataspacedAdminForm @@ -47,6 +51,7 @@ from product_portfolio.models import Product from product_portfolio.models import ProductComponent from product_portfolio.models import ProductPackage +from product_portfolio.models import ScanCodeProject class NameVersionValidationFormMixin: @@ -560,6 +565,27 @@ def helper(self): helper.add_input(Submit("import", "Import")) return helper + def save(self, product): + from product_portfolio.importers import ImportFromScan + + sid = transaction.savepoint() + importer = ImportFromScan( + product, + self.user, + upload_file=self.cleaned_data.get("upload_file"), + create_codebase_resources=self.cleaned_data.get("create_codebase_resources"), + stop_on_error=self.cleaned_data.get("stop_on_error"), + ) + + try: + warnings, created_counts = importer.save() + except ValidationError: + transaction.savepoint_rollback(sid) + raise + + transaction.savepoint_commit(sid) + return warnings, created_counts + class LoadSBOMsForm(forms.Form): input_file = SmartFileField( @@ -597,6 +623,24 @@ def helper(self): helper.add_input(Submit("submit", "Load Packages", css_class="btn-success")) return helper + def submit(self, product, user): + scancode_project = ScanCodeProject.objects.create( + product=product, + dataspace=product.dataspace, + type=ScanCodeProject.ProjectType.LOAD_SBOMS, + input_file=self.cleaned_data.get("input_file"), + update_existing_packages=self.cleaned_data.get("update_existing_packages"), + scan_all_packages=self.cleaned_data.get("scan_all_packages"), + created_by=user, + ) + + transaction.on_commit( + lambda: tasks.scancodeio_submit_load_sbom.delay( + scancodeproject_uuid=scancode_project.uuid, + user_uuid=user.uuid, + ) + ) + class StrongTextWidget(forms.Widget): def render(self, name, value, attrs=None, renderer=None): @@ -847,3 +891,35 @@ def helper(self): helper.form_id = "pull-project-data-form" helper.attrs = {"autocomplete": "off"} return helper + + def get_project_data(self, project_name_or_uuid, user): + scancodeio = ScanCodeIO(user) + for field_name in ["name", "uuid"]: + project_data = scancodeio.find_project(**{field_name: project_name_or_uuid}) + if project_data: + return project_data + + def submit(self, product, user): + project_name_or_uuid = self.cleaned_data.get("project_name_or_uuid") + project_data = self.get_project_data(project_name_or_uuid, user) + + if not project_data: + msg = f'Project "{project_name_or_uuid}" not found on ScanCode.io.' + raise ValidationError(msg) + + scancode_project = ScanCodeProject.objects.create( + product=product, + dataspace=product.dataspace, + type=ScanCodeProject.ProjectType.PULL_FROM_SCANCODEIO, + project_uuid=project_data.get("uuid"), + update_existing_packages=self.cleaned_data.get("update_existing_packages"), + scan_all_packages=False, + status=ScanCodeProject.Status.SUBMITTED, + created_by=user, + ) + + transaction.on_commit( + lambda: tasks.pull_project_data_from_scancodeio.delay( + scancodeproject_uuid=scancode_project.uuid, + ) + ) diff --git a/product_portfolio/templates/product_portfolio/import_from_scan.html b/product_portfolio/templates/product_portfolio/import_from_scan.html index 0236bdc8..8e702e93 100644 --- a/product_portfolio/templates/product_portfolio/import_from_scan.html +++ b/product_portfolio/templates/product_portfolio/import_from_scan.html @@ -50,8 +50,11 @@
Option 2: From ScanCode.io pipeline results
Upload a ScanCode.io JSON output file, generated with one of the following pipelines:

- docker, docker_windows, inspect_manifest, load_inventory - root_filesystems, scan_codebase, scan_package + analyze_docker_image, + analyze_windows_docker_image, + inspect_packages, + scan_codebase, + scan_single_package

diff --git a/product_portfolio/tests/test_api.py b/product_portfolio/tests/test_api.py index 46c270ab..9001aabf 100644 --- a/product_portfolio/tests/test_api.py +++ b/product_portfolio/tests/test_api.py @@ -7,8 +7,12 @@ # import json +import uuid +from pathlib import Path +from unittest import mock from django.core import mail +from django.core.files.base import ContentFile from django.test import TestCase from django.urls import reverse @@ -41,9 +45,12 @@ from product_portfolio.models import ProductPackage from product_portfolio.models import ProductRelationStatus from product_portfolio.models import ProductStatus +from product_portfolio.models import ScanCodeProject class ProductAPITestCase(MaxQueryMixin, TestCase): + testfiles_path = Path(__file__).parent / "testfiles" + def setUp(self): self.dataspace = Dataspace.objects.create(name="nexB") self.alternate_dataspace = Dataspace.objects.create(name="Alternate") @@ -343,6 +350,113 @@ def test_api_product_endpoint_update_permissions(self): product1 = Product.unsecured_objects.get(pk=self.product1.pk) self.assertEqual("Updated Name", product1.name) + def test_api_product_endpoint_load_sboms_action(self): + url = reverse("api_v2:product-load-sboms", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, response.status_code) + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_403_FORBIDDEN, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + expected = {"input_file": ["This field is required."]} + self.assertEqual(expected, response.data) + + data = { + "input_file": ContentFile("Content", name="sbom.json"), + "update_existing_packages": False, + "scan_all_packages": False, + } + response = self.client.post(url, data) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = {"status": "SBOM file submitted to ScanCode.io for inspection."} + self.assertEqual(expected, response.data) + self.assertEqual(1, ScanCodeProject.objects.count()) + + def test_api_product_endpoint_import_from_scan_action(self): + url = reverse("api_v2:product-import-from-scan", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, response.status_code) + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_403_FORBIDDEN, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + expected = {"upload_file": ["This field is required."]} + self.assertEqual(expected, response.data) + + data = { + "upload_file": ContentFile("Content", name="scan_results.json"), + "create_codebase_resources": False, + "stop_on_error": False, + } + response = self.client.post(url, data) + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + expected = ["The file content is not proper JSON."] + self.assertEqual(expected, response.data) + + scan_input_location = self.testfiles_path / "import_from_scan.json" + data = { + "upload_file": scan_input_location.open(), + "create_codebase_resources": True, + "stop_on_error": False, + } + response = self.client.post(url, data) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = { + "status": "Imported from Scan: 1 Packages, 1 Product Packages, 3 Codebase Resources" + } + self.assertEqual(1, self.product1.productpackages.count()) + self.assertEqual(1, self.product1.packages.count()) + self.assertEqual(3, self.product1.codebaseresources.count()) + + @mock.patch("product_portfolio.forms.PullProjectDataForm.get_project_data") + def test_api_product_endpoint_pull_scancodeio_project_data_action(self, mock_get_project_data): + url = reverse("api_v2:product-pull-scancodeio-project-data", args=[self.product1.uuid]) + + self.client.login(username=self.base_user.username, password="secret") + response = self.client.get(url) + self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, response.status_code) + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_403_FORBIDDEN, response.status_code) + + # Required permissions + add_perm(self.base_user, "add_product") + assign_perm("view_product", self.base_user, self.product1) + + response = self.client.post(url, data={}) + self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) + expected = {"project_name_or_uuid": ["This field is required."]} + self.assertEqual(expected, response.data) + + mock_get_project_data.return_value = None + data = { + "project_name_or_uuid": "project_name", + "update_existing_packages": False, + } + response = self.client.post(url, data) + expected = ['Project "project_name" not found on ScanCode.io.'] + self.assertEqual(expected, response.data) + + mock_get_project_data.return_value = {"uuid": uuid.uuid4()} + response = self.client.post(url, data) + self.assertEqual(status.HTTP_200_OK, response.status_code) + expected = {"status": "Packages import from ScanCode.io in progress..."} + self.assertEqual(expected, response.data) + self.assertEqual(1, ScanCodeProject.objects.count()) + class ProductRelatedAPITestCase(TestCase): def setUp(self): diff --git a/product_portfolio/tests/test_importers.py b/product_portfolio/tests/test_importers.py index b72fc99b..d370194a 100644 --- a/product_portfolio/tests/test_importers.py +++ b/product_portfolio/tests/test_importers.py @@ -26,6 +26,7 @@ from license_library.models import LicenseChoice from organization.models import Owner from product_portfolio.importers import CodebaseResourceImporter +from product_portfolio.importers import ImportFromScan from product_portfolio.importers import ImportPackageFromScanCodeIO from product_portfolio.importers import ProductComponentImporter from product_portfolio.importers import ProductPackageImporter @@ -35,7 +36,6 @@ from product_portfolio.models import ProductItemPurpose from product_portfolio.models import ProductPackage from product_portfolio.models import ProductRelationStatus -from product_portfolio.views import ImportFromScan class ProductRelationImporterTestCase(TestCase): diff --git a/product_portfolio/views.py b/product_portfolio/views.py index 46653a62..c8b5b56d 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -109,7 +109,6 @@ from product_portfolio.forms import ProductPackageInlineForm from product_portfolio.forms import PullProjectDataForm from product_portfolio.forms import TableInlineFormSetHelper -from product_portfolio.importers import ImportFromScan from product_portfolio.models import CodebaseResource from product_portfolio.models import Product from product_portfolio.models import ProductComponent @@ -1483,23 +1482,12 @@ def import_from_scan_view(request, dataspace, name, version=""): files=request.FILES, ) if form.is_valid(): - sid = transaction.savepoint() - importer = ImportFromScan( - product, - user, - upload_file=form.cleaned_data.get("upload_file"), - create_codebase_resources=form.cleaned_data.get("create_codebase_resources"), - stop_on_error=form.cleaned_data.get("stop_on_error"), - ) try: - warnings, created_counts = importer.save() + warnings, created_counts = form.save(product=product) except ValidationError as error: - transaction.savepoint_rollback(sid) messages.error(request, " ".join(error.messages)) return redirect(request.path) - transaction.savepoint_commit(sid) - if not created_counts: messages.warning(request, "Nothing imported.") else: @@ -1882,24 +1870,7 @@ def get_success_url(self): def form_valid(self, form): self.object = self.get_object() - - scancode_project = ScanCodeProject.objects.create( - product=self.object, - dataspace=self.object.dataspace, - type=ScanCodeProject.ProjectType.LOAD_SBOMS, - input_file=form.cleaned_data.get("input_file"), - update_existing_packages=form.cleaned_data.get("update_existing_packages"), - scan_all_packages=form.cleaned_data.get("scan_all_packages"), - created_by=self.request.user, - ) - - transaction.on_commit( - lambda: tasks.scancodeio_submit_load_sbom.delay( - scancodeproject_uuid=scancode_project.uuid, - user_uuid=self.request.user.uuid, - ) - ) - + form.submit(product=self.object, user=self.request.user) msg = "SBOM file submitted to ScanCode.io for inspection." messages.success(self.request, msg) return super().form_valid(form) @@ -1990,40 +1961,15 @@ def form_invalid(self, form): def get_success_url(self): return f"{self.object.get_absolute_url()}#imports" - def get_project_data(self, project_name_or_uuid): - scancodeio = ScanCodeIO(self.request.user) - for field_name in ["name", "uuid"]: - project_data = scancodeio.find_project(**{field_name: project_name_or_uuid}) - if project_data: - return project_data - def form_valid(self, form): - project_name_or_uuid = form.cleaned_data.get("project_name_or_uuid") - project_data = self.get_project_data(project_name_or_uuid) + self.object = self.get_object() - if not project_data: - msg = f'Project "{project_name_or_uuid}" not found on ScanCode.io.' - messages.error(self.request, msg) + try: + form.submit(product=self.object, user=self.request.user) + except ValidationError as error: + messages.error(self.request, error) return redirect(self.object.get_absolute_url()) - scancode_project = ScanCodeProject.objects.create( - product=self.object, - dataspace=self.object.dataspace, - type=ScanCodeProject.ProjectType.PULL_FROM_SCANCODEIO, - project_uuid=project_data.get("uuid"), - update_existing_packages=form.cleaned_data.get("update_existing_packages"), - scan_all_packages=False, - status=ScanCodeProject.Status.SUBMITTED, - created_by=self.request.user, - ) - - transaction.on_commit( - lambda: tasks.pull_project_data_from_scancodeio.delay( - scancodeproject_uuid=scancode_project.uuid, - ) - ) - - project_name = project_data.get("name") - msg = f'Packages import from ScanCode.io "{project_name}" in progress...' + msg = "Packages import from ScanCode.io in progress..." messages.success(self.request, msg) return super().form_valid(form)