8000 Add a "Load Packages from SBOMs" Product action in the REST API #59 by tdruez · Pull Request #62 · aboutcode-org/dejacode · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Add a "Load Packages from SBOMs" Product action in the REST API #59 #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ Release notes
- Add dark theme support in UI.
https://github.com/nexB/dejacode/issues/25

- Add "Load Packages from SBOMs", "Import scan results", and
"Pull ScanCode.io project data" feature as Product action in the REST API.
https://github.com/nexB/dejacode/issues/59

- Refactor the "Import manifest" feature as "Load SBOMs".
https://github.com/nexB/dejacode/issues/61

### Version 5.0.1

- Improve the stability of the "Check for new Package versions" feature.
Expand Down
123 changes: 123 additions & 0 deletions product_portfolio/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@
import django_filters
from rest_framework import permissions
from rest_framework import serializers
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.permissions import SAFE_METHODS
from rest_framework.response import Response

from component_catalog.api import KeywordsField
from component_catalog.api import PackageEmbeddedSerializer
Expand All @@ -31,6 +34,9 @@
from dje.filters import NameVersionFilter
from dje.permissions import assign_all_object_permissions
from product_portfolio.filters import ComponentCompletenessAPIFilter
from product_portfolio.forms import ImportFromScanForm
from product_portfolio.forms import LoadSBOMsForm
from product_portfolio.forms import PullProjectDataForm
from product_portfolio.models import CodebaseResource
from product_portfolio.models import Product
from product_portfolio.models import ProductComponent
Expand Down Expand Up @@ -191,6 +197,57 @@ class Meta:
)


class LoadSBOMsFormSerializer(serializers.Serializer):
"""Serializer equivalent of LoadSBOMsForm, used for API documentation."""

input_file = serializers.FileField(
required=True,
help_text=LoadSBOMsForm.base_fields["input_file"].label,
)
update_existing_packages = serializers.BooleanField(
required=False,
default=False,
help_text=LoadSBOMsForm.base_fields["update_existing_packages"].help_text,
)
scan_all_packages = serializers.BooleanField(
required=False,
default=False,
help_text=LoadSBOMsForm.base_fields["scan_all_packages"].help_text,
)


class ImportFromScanSerializer(serializers.Serializer):
"""Serializer equivalent of ImportFromScanForm, used for API documentation."""

upload_file = serializers.FileField(
required=True,
)
create_codebase_resources = serializers.BooleanField(
required=False,
default=False,
help_text=ImportFromScanForm.base_fields["create_codebase_resources"].help_text,
)
stop_on_error = serializers.BooleanField(
required=False,
default=False,
help_text=ImportFromScanForm.base_fields["stop_on_error"].help_text,
)


class PullProjectDataSerializer(serializers.Serializer):
"""Serializer equivalent of PullProjectDataForm, used for API documentation."""

project_name_or_uuid = serializers.CharField(
required=True,
help_text=PullProjectDataForm.base_fields["project_name_or_uuid"].label,
)
update_existing_packages = serializers.BooleanField(
required=False,
default=False,
help_text=PullProjectDataForm.base_fields["update_existing_packages"].help_text,
)


class ProductViewSet(CreateRetrieveUpdateListViewSet):
queryset = Product.objects.none()
serializer_class = ProductSerializer
Expand Down Expand Up @@ -240,6 +297,72 @@ def perform_create(self, serializer):
super().perform_create(serializer)
assign_all_object_permissions(self.request.user, serializer.instance)

@action(detail=True, methods=["post"], serializer_class=LoadSBOMsFormSerializer)
def load_sboms(self, request, *args, **kwargs):
"""
Load Packages from SBOMs.

DejaCode supports the following SBOM formats:
* CycloneDX BOM as JSON bom.json and .cdx.json,
* SPDX document as JSON .spdx.json,
* AboutCode .ABOUT files,

Multiple SBOMs: You can provide multiple SBOMs by packaging them into a zip
archive. DejaCode will handle and process them accordingly.
"""
product = self.get_object()

form = LoadSBOMsForm(data=request.POST, files=request.FILES)
if not form.is_valid():
return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)

form.submit(product=product, user=request.user)
return Response({"status": "SBOM file submitted to ScanCode.io for inspection."})

@action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer)
def import_from_scan(self, request, *args, **kwargs):
"""
Import the scan results in the Product.

Upload a ScanCode.io or ScanCode-toolkit JSON output file.
"""
product = self.get_object()

8000 form = ImportFromScanForm(user=request.user, data=request.POST, files=request.FILES)
if not form.is_valid():
return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)

try:
warnings, created_counts = form.save(product=product)
except ValidationError as error:
return Response(error.messages, status=status.HTTP_400_BAD_REQUEST)

if not created_counts:
msg = "Nothing imported."
else:
msg = "Imported from Scan: "
msg += ", ".join([f"{value} {key}" for key, value in created_counts.items()])
return Response({"status": msg})

@action(detail=True, methods=["post"], serializer_class=PullProjectDataSerializer)
def pull_scancodeio_project_data(self, request, *args, **kwargs):
"""
Pull data from a ScanCode.io Project to import all its Discovered Packages.
Imported Packages will be assigned to this Product.
"""
product = self.get_object()

form = PullProjectDataForm(data=request.POST)
if not form.is_valid():
return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)

try:
form.submit(product=product, user=request.user)
except ValidationError as error:
return Response(error.messages, status=status.HTTP_400_BAD_REQUEST)

return Response({"status": "Packages import from ScanCode.io in progress..."})


class BaseProductRelationSerializer(ValidateLicenseExpressionMixin, DataspacedSerializer):
product = NameVersionHyperlinkedRelatedField(
Expand Down
76 changes: 76 additions & 0 deletions product_portfolio/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#

from django import forms
from django.core.exceptions import ValidationError
from django.db import transaction
from django.forms import BaseModelFormSet
from django.forms.formsets import DELETION_FIELD_NAME
from django.urls import reverse_lazy
Expand All @@ -28,6 +30,8 @@
from component_catalog.license_expression_dje import LicenseExpressionFormMixin
from component_catalog.models import Component
from component_catalog.programming_languages import PROGRAMMING_LANGUAGES
from dejacode_toolkit.scancodeio import ScanCodeIO
from dje import tasks
from dje.fields import SmartFileField
from dje.forms import ColorCodeFormMixin
from dje.forms import DataspacedAdminForm
Expand All @@ -47,6 +51,7 @@
from product_portfolio.models import Product
from product_portfolio.models import ProductComponent
from product_portfolio.models import ProductPackage
from product_portfolio.models import ScanCodeProject


class NameVersionValidationFormMixin:
Expand Down Expand Up @@ -560,6 +565,27 @@ def helper(self):
helper.add_input(Submit("import", "Import"))
return helper

def save(self, product):
from product_portfolio.importers import ImportFromScan

sid = tr 6D40 ansaction.savepoint()
importer = ImportFromScan(
product,
self.user,
upload_file=self.cleaned_data.get("upload_file"),
create_codebase_resources=self.cleaned_data.get("create_codebase_resources"),
stop_on_error=self.cleaned_data.get("stop_on_error"),
)

try:
warnings, created_counts = importer.save()
except ValidationError:
transaction.savepoint_rollback(sid)
raise

transaction.savepoint_commit(sid)
return warnings, created_counts


class LoadSBOMsForm(forms.Form):
input_file = SmartFileField(
Expand Down Expand Up @@ -597,6 +623,24 @@ def helper(self):
helper.add_input(Submit("submit", "Load Packages", css_class="btn-success"))
return helper

def submit(self, product, user):
scancode_project = ScanCodeProject.objects.create(
product=product,
dataspace=product.dataspace,
type=ScanCodeProject.ProjectType.LOAD_SBOMS,
input_file=self.cleaned_data.get("input_file"),
update_existing_packages=self.cleaned_data.get("update_existing_packages"),
scan_all_packages=self.cleaned_data.get("scan_all_packages"),
created_by=user,
)

transaction.on_commit(
lambda: tasks.scancodeio_submit_load_sbom.delay(
scancodeproject_uuid=scancode_project.uuid,
user_uuid=user.uuid,
)
)


class StrongTextWidget(forms.Widget):
def render(self, name, value, attrs=None, renderer=None):
Expand Down Expand Up @@ -847,3 +891,35 @@ def helper(self):
helper.form_id = "pull-project-data-form"
helper.attrs = {"autocomplete": "off"}
return helper

def get_project_data(self, project_name_or_uuid, user):
scancodeio = ScanCodeIO(user)
for field_name in ["name", "uuid"]:
project_data = scancodeio.find_project(**{field_name: project_name_or_uuid})
if project_data:
return project_data

def submit(self, product, user):
project_name_or_uuid = self.cleaned_data.get("project_name_or_uuid")
project_data = self.get_project_data(project_name_or_uuid, user)

if not project_data:
msg = f'Project "{project_name_or_uuid}" not found on ScanCode.io.'
raise ValidationError(msg)

scancode_project = ScanCodeProject.objects.create(
product=product,
dataspace=product.dataspace,
type=ScanCodeProject.ProjectType.PULL_FROM_SCANCODEIO,
project_uuid=project_data.get("uuid"),
update_existing_packages=self.cleaned_data.get("update_existing_packages"),
scan_all_packages=False,
status=ScanCodeProject.Status.SUBMITTED,
created_by=user,
)

transaction.on_commit(
lambda: tasks.pull_project_data_from_scancodeio.delay(
scancodeproject_uuid=scancode_project.uuid,
)
)
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,11 @@ <h5>Option 2: From ScanCode.io pipeline results</h5>
Upload a ScanCode.io JSON output file, <strong>generated with one of the following pipelines:</strong>
</p>
<p class="mb-0">
<code>docker</code>, <code>docker_windows</code>, <code>inspect_manifest</code>, <code>load_inventory</code>
<code>root_filesystems</code>, <code>scan_codebase</code>, <code>scan_package</code>
<code>analyze_docker_image</code>,
<code>analyze_windows_docker_image</code>,
<code>inspect_packages</code>,
<code>scan_codebase</code>,
<code>scan_single_package</code>
</p>
</div>

Expand Down
Loading
0