Uncaught Exception from Nessus Scan Import without Filename

[tohch4]

Per our conversation in OWASP Slack, I am trying to import the sample scan file without our evaluation deployment of v.1.5.4rc6 as we slowly move up changes. Using the current master branch of the code in defectdojo_api, we encounter an uncaught exception and our Nessus imports do not work.

uwsgi_1         | Internal Server Error: /api/v2/import-scan/
uwsgi_1         | Traceback (most recent call last):
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/exception.py", line 34, in inner
uwsgi_1         |     response = get_response(request)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 115, in _get_response
uwsgi_1         |     response = self.process_exception_by_middleware(e, request)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 113, in _get_response
uwsgi_1         |     response = wrapped_callback(request, *callback_args, **callback_kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
uwsgi_1         |     return view_func(*args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/viewsets.py", line 114, in view
uwsgi_1         |     return self.dispatch(request, *args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 505, in dispatch
uwsgi_1         |     response = self.handle_exception(exc)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 465, in handle_exception
uwsgi_1         |     self.raise_uncaught_exception(exc)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
uwsgi_1         |     raise exc
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 502, in dispatch
uwsgi_1         |     response = handler(request, *args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/mixins.py", line 19, in create
uwsgi_1         |     self.perform_create(serializer)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/mixins.py", line 24, in perform_create
uwsgi_1         |     serializer.save()
uwsgi_1         |   File "./dojo/api_v2/serializers.py", line 576, in save
uwsgi_1         |     data['scan_type'],)
uwsgi_1         |   File "./dojo/tools/factory.py", line 233, in import_parser_factory
uwsgi_1         |     return parser
uwsgi_1         | UnboundLocalError: local variable 'parser' referenced before assignment
uwsgi_1         | Internal Server Error: /api/v2/import-scan/
uwsgi_1         | Traceback (most recent call last):
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/exception.py", line 34, in inner
uwsgi_1         |     response = get_response(request)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 115, in _get_response
uwsgi_1         |     response = self.process_exception_by_middleware(e, request)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 113, in _get_response
uwsgi_1         |     response = wrapped_callback(request, *callback_args, **callback_kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
uwsgi_1         |     return view_func(*args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/viewsets.py", line 114, in view
uwsgi_1         |     return self.dispatch(request, *args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 505, in dispatch
uwsgi_1         |     response = self.handle_exception(exc)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 465, in handle_exception
uwsgi_1         |     self.raise_uncaught_exception(exc)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
uwsgi_1         |     raise exc
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 502, in dispatch
uwsgi_1         |     response = handler(request, *args, **kwargs)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/mixins.py", line 19, in create
uwsgi_1         |     self.perform_create(serializer)
uwsgi_1         |   File "/usr/local/lib/python3.5/site-packages/rest_framework/mixins.py", line 24, in perform_create
uwsgi_1         |     serializer.save()
uwsgi_1         |   File "./dojo/api_v2/serializers.py", line 576, in save
uwsgi_1         |     data['scan_type'],)
uwsgi_1         |   File "./dojo/tools/factory.py", line 233, in import_parser_factory
uwsgi_1         |     return parser
uwsgi_1         | UnboundLocalError: local variable 'parser' referenced before assignment

The UnboundLocalError is caused by importing this .nessus file from the sample_scan_file collection for unit testing using the following script that imports said file with the defectdojo_api.

import importlib
import json
import logging
import os
#import boto3
from datetime import date, timedelta
from defectdojo_api.defectdojo_api import defectdojo_apiv2 as defectdojo
import json
REPORT_TYPE = 'Nessus Scan'
ACTIVE = True
VERIFIED = False
CLOSE_OLD_FINDINGS = True
SKIP_DUPLICATES = True
LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
def dd_connection():
     # setup DefectDojo connection information
    dd_host = os.environ['dd_host']
    api_token = os.environ['api_token']
    user = os.environ['dd_user']
    # instantiate the DefectDojo api wrapper
    connection_obj = defectdojo.DefectDojoAPIv2(dd_host, api_token, user, debug=False, verify_ssl=False)
    return connection_obj
dd = dd_connection()
d = date.today()
report_date = d.strftime("%Y-%m-%d")
test = dd.upload_scan(2, REPORT_TYPE, 'local/path/to/DefectDojo/sample-scan-files/nessus/nessus_v_unknown.nessus', ACTIVE, VERIFIED, CLOSE_OLD_FINDINGS, SKIP_DUPLICATES, report_date, tags="Lambda")
print(test.message)
# product_id = get_projects(dd)
# engagements = dd.list_engagements(product_id=product_id, limit=100)

By stepping through the debugger it appears that the DefectDojo API attempts to conditionally parse the file based on the extension of the filename provided. The relevant function in defectdojo_api uses the requests library and the function style used here will not pass in the filename in the Content-Disposition header as explained here.

    def upload_scan(self, engagement_id, scan_type, file, active, verified, close_old_findings, skip_duplicates, scan_date, tags=None, build=None, minimum_severity="Info"):
        """Uploads and processes a scan file.
        :param application_id: Application identifier.
        :param file_path: Path to the scan file to be uploaded.
        """
        if tags is None:
            tags = ''

        if build is None:
            build = ''

        with open(file, 'rb') as f:
             filedata = f.read()
        
        if self.debug:
            print("filedata:")
            print(filedata)

        data = {
            'file': filedata,
            'engagement': ('', engagement_id),
            'scan_type': ('', scan_type),
            'active': ('', active),
            'verified': ('', verified),
            'close_old_findings': ('', close_old_findings),
            'skip_duplicates': ('', skip_duplicates),
            'scan_date': ('', scan_date),
            'tags': ('', tags),
            'build_id': ('', build),
            'minimum_severity': ('', minimum_severity)
        }
        """
        TODO: implement these parameters:
          lead
          test_type
          scan_date
        """

        return self._request(
            'POST', 'import-scan/',
            files=data
        )

Changing the code to this style appears to resolve the issue, where we adapt the script about to pass in the data and filename into a tuple, and pass that into the upload_scan() function, and that dd_api upload_scan() function does not open the file and buffer itself. This way, the file data and name is passed.

    def upload_scan(self, engagement_id, scan_type, filedata, active, verified, close_old_findings, skip_duplicates, scan_date, tags=None, build=None, minimum_severity="Info"):
        """Uploads and processes a scan file.

        :param application_id: Application identifier.
        :param file: Tuple with name and contents of (filename, open(filename, 'rb'))

        """
        if tags is None:
            tags = ''

        if build is None:
            build = ''
        
        if self.debug:
            print("filedata:")
            print(filedata)

        data = {
            'file': filedata,
            'engagement': ('', engagement_id),
            'scan_type': ('', scan_type),
            'active': ('', active),
            'verified': ('', verified),
            'close_old_findings': ('', close_old_findings),
            'skip_duplicates': ('', skip_duplicates),
            'scan_date': ('', scan_date),
            'tags': ('', tags),
            'build_id': ('', build),
            'minimum_severity': ('', minimum_severity)
        }
        """
        TODO: implement these parameters:
          lead
          test_type
          scan_date
        """

        return self._request(
            'POST', 'import-scan/',
            files=data
        )

import importlib
import json
import logging
import os
#import boto3
from datetime import date, timedelta
from defectdojo_api.defectdojo_api import defectdojo_apiv2 as defectdojo
import json
REPORT_TYPE = 'Nessus Scan'
ACTIVE = True
VERIFIED = False
CLOSE_OLD_FINDINGS = True
SKIP_DUPLICATES = True
LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
def dd_connection():
     # setup DefectDojo connection information
    dd_host = os.environ['dd_host']
    api_token = os.environ['api_token']
    user = os.environ['dd_user']
    # instantiate the DefectDojo api wrapper
    connection_obj = defectdojo.DefectDojoAPIv2(dd_host, api_token, user, debug=False, verify_ssl=False)
    return connection_obj
dd = dd_connection()
d = date.today()
report_date = d.strftime("%Y-%m-%d")
test = dd.upload_scan(2, REPORT_TYPE, ('nessus_v_unknown.nessus', open('local/path/to/DefectDojo/sample-scan-files/nessus/nessus_v_unknown.nessus', 'rb')), ACTIVE, VERIFIED, CLOSE_OLD_FINDINGS, SKIP_DUPLICATES, report_date, tags="Lambda")
print(test.message)
# product_id = get_projects(dd)
# engagements = dd.list_engagements(product_id=product_id, limit=100)

If amenable to this approach, I will draft a PR to fix this issue.

[tohch4]

Given the lack of feedback, I will move forward with the PR and discuss in the newly published #defectdojo-dev channel in OWASP Slack.

[valentijnscholten]

PRs are always welcome. It could be that nobody on our side ever really understood the issue as it is working for us. At least with the scan types we're using, also the main Defect Dojo repo is monitored better compared to the api repo.