mmdetection/mmdet/evaluation/functional/cityscapes_utils.py

# Copyright (c) OpenMMLab. All rights reserved.
# Copyright (c) https://github.com/mcordts/cityscapesScripts
# A wrapper of `cityscapesscripts` which supports loading groundtruth
# image from `backend_args`.
import json
import os
import sys
from pathlib import Path
from typing import Optional, Union

import mmcv
import numpy as np
from mmengine.fileio import get

try:
    import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as CSEval  # noqa: E501
    from cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling import \
        CArgs  # noqa: E501
    from cityscapesscripts.evaluation.instance import Instance
    from cityscapesscripts.helpers.csHelpers import (id2label, labels,
                                                     writeDict2JSON)
    HAS_CITYSCAPESAPI = True
except ImportError:
    CArgs = object
    HAS_CITYSCAPESAPI = False


def evaluateImgLists(prediction_list: list,
                     groundtruth_list: list,
                     args: CArgs,
                     backend_args: Optional[dict] = None,
                     dump_matches: bool = False) -> dict:
    """A wrapper of obj:``cityscapesscripts.evaluation.

    evalInstanceLevelSemanticLabeling.evaluateImgLists``. Support loading
    groundtruth image from file backend.
    Args:
        prediction_list (list): A list of prediction txt file.
        groundtruth_list (list): A list of groundtruth image file.
        args (CArgs): A global object setting in
            obj:``cityscapesscripts.evaluation.
            evalInstanceLevelSemanticLabeling``
        backend_args (dict, optional): Arguments to instantiate the
            preifx of uri corresponding backend. Defaults to None.
        dump_matches (bool): whether dump matches.json. Defaults to False.
    Returns:
        dict: The computed metric.
    """
    if not HAS_CITYSCAPESAPI:
        raise RuntimeError('Failed to import `cityscapesscripts`.'
                           'Please try to install official '
                           'cityscapesscripts by '
                           '"pip install cityscapesscripts"')
    # determine labels of interest
    CSEval.setInstanceLabels(args)
    # get dictionary of all ground truth instances
    gt_instances = getGtInstances(
        groundtruth_list, args, backend_args=backend_args)
    # match predictions and ground truth
    matches = matchGtWithPreds(prediction_list, groundtruth_list, gt_instances,
                               args, backend_args)
    if dump_matches:
        CSEval.writeDict2JSON(matches, 'matches.json')
    # evaluate matches
    apScores = CSEval.evaluateMatches(matches, args)
    # averages
    avgDict = CSEval.computeAverages(apScores, args)
    # result dict
    resDict = CSEval.prepareJSONDataForResults(avgDict, apScores, args)
    if args.JSONOutput:
        # create output folder if necessary
        path = os.path.dirname(args.exportFile)
        CSEval.ensurePath(path)
        # Write APs to JSON
        CSEval.writeDict2JSON(resDict, args.exportFile)

    CSEval.printResults(avgDict, args)

    return resDict


def matchGtWithPreds(prediction_list: list,
                     groundtruth_list: list,
                     gt_instances: dict,
                     args: CArgs,
                     backend_args=None):
    """A wrapper of obj:``cityscapesscripts.evaluation.

    evalInstanceLevelSemanticLabeling.matchGtWithPreds``. Support loading
    groundtruth image from file backend.
    Args:
        prediction_list (list): A list of prediction txt file.
        groundtruth_list (list): A list of groundtruth image file.
        gt_instances (dict): Groundtruth dict.
        args (CArgs): A global object setting in
            obj:``cityscapesscripts.evaluation.
            evalInstanceLevelSemanticLabeling``
        backend_args (dict, optional): Arguments to instantiate the
            preifx of uri corresponding backend. Defaults to None.
    Returns:
        dict: The processed prediction and groundtruth result.
    """
    if not HAS_CITYSCAPESAPI:
        raise RuntimeError('Failed to import `cityscapesscripts`.'
                           'Please try to install official '
                           'cityscapesscripts by '
                           '"pip install cityscapesscripts"')
    matches: dict = dict()
    if not args.quiet:
        print(f'Matching {len(prediction_list)} pairs of images...')

    count = 0
    for (pred, gt) in zip(prediction_list, groundtruth_list):
        # Read input files
        gt_image = readGTImage(gt, backend_args)
        pred_info = readPredInfo(pred)
        # Get and filter ground truth instances
        unfiltered_instances = gt_instances[gt]
        cur_gt_instances_orig = CSEval.filterGtInstances(
            unfiltered_instances, args)

        # Try to assign all predictions
        (cur_gt_instances,
         cur_pred_instances) = CSEval.assignGt2Preds(cur_gt_instances_orig,
                                                     gt_image, pred_info, args)

        # append to global dict
        matches[gt] = {}
        matches[gt]['groundTruth'] = cur_gt_instances
        matches[gt]['prediction'] = cur_pred_instances

        count += 1
        if not args.quiet:
            print(f'\rImages Processed: {count}', end=' ')
            sys.stdout.flush()

    if not args.quiet:
        print('')

    return matches


def readGTImage(image_file: Union[str, Path],
                backend_args: Optional[dict] = None) -> np.ndarray:
    """Read an image from path.

    Same as obj:``cityscapesscripts.evaluation.
    evalInstanceLevelSemanticLabeling.readGTImage``, but support loading
    groundtruth image from file backend.
    Args:
        image_file (str or Path): Either a str or pathlib.Path.
        backend_args (dict, optional): Instantiates the corresponding file
            backend. It may contain `backend` key to specify the file
            backend. If it contains, the file backend corresponding to this
            value will be used and initialized with the remaining values,
            otherwise the corresponding file backend will be selected
            based on the prefix of the file path. Defaults to None.
    Returns:
        np.ndarray: The groundtruth image.
    """
    img_bytes = get(image_file, backend_args=backend_args)
    img = mmcv.imfrombytes(img_bytes, flag='unchanged', backend='pillow')
    return img


def readPredInfo(prediction_file: str) -> dict:
    """A wrapper of obj:``cityscapesscripts.evaluation.

    evalInstanceLevelSemanticLabeling.readPredInfo``.
    Args:
        prediction_file (str): The prediction txt file.
    Returns:
        dict: The processed prediction results.
    """
    if not HAS_CITYSCAPESAPI:
        raise RuntimeError('Failed to import `cityscapesscripts`.'
                           'Please try to install official '
                           'cityscapesscripts by '
                           '"pip install cityscapesscripts"')
    printError = CSEval.printError

    predInfo = {}
    if (not os.path.isfile(prediction_file)):
        printError(f"Infofile '{prediction_file}' "
                   'for the predictions not found.')
    with open(prediction_file) as f:
        for line in f:
            splittedLine = line.split(' ')
            if len(splittedLine) != 3:
                printError('Invalid prediction file. Expected content: '
                           'relPathPrediction1 labelIDPrediction1 '
                           'confidencePrediction1')
            if os.path.isabs(splittedLine[0]):
                printError('Invalid prediction file. First entry in each '
                           'line must be a relative path.')

            filename = os.path.join(
                os.path.dirname(prediction_file), splittedLine[0])

            imageInfo = {}
            imageInfo['labelID'] = int(float(splittedLine[1]))
            imageInfo['conf'] = float(splittedLine[2])  # type: ignore
            predInfo[filename] = imageInfo

    return predInfo


def getGtInstances(groundtruth_list: list,
                   args: CArgs,
                   backend_args: Optional[dict] = None) -> dict:
    """A wrapper of obj:``cityscapesscripts.evaluation.

    evalInstanceLevelSemanticLabeling.getGtInstances``. Support loading
    groundtruth image from file backend.
    Args:
        groundtruth_list (list): A list of groundtruth image file.
        args (CArgs): A global object setting in
            obj:``cityscapesscripts.evaluation.
            evalInstanceLevelSemanticLabeling``
        backend_args (dict, optional): Arguments to instantiate the
            preifx of uri corresponding backend. Defaults to None.
    Returns:
        dict: The computed metric.
    """
    if not HAS_CITYSCAPESAPI:
        raise RuntimeError('Failed to import `cityscapesscripts`.'
                           'Please try to install official '
                           'cityscapesscripts by '
                           '"pip install cityscapesscripts"')
    # if there is a global statistics json, then load it
    if (os.path.isfile(args.gtInstancesFile)):
        if not args.quiet:
            print('Loading ground truth instances from JSON.')
        with open(args.gtInstancesFile) as json_file:
            gt_instances = json.load(json_file)
    # otherwise create it
    else:
        if (not args.quiet):
            print('Creating ground truth instances from png files.')
        gt_instances = instances2dict(
            groundtruth_list, args, backend_args=backend_args)
        writeDict2JSON(gt_instances, args.gtInstancesFile)

    return gt_instances


def instances2dict(image_list: list,
                   args: CArgs,
                   backend_args: Optional[dict] = None) -> dict:
    """A wrapper of obj:``cityscapesscripts.evaluation.

    evalInstanceLevelSemanticLabeling.instances2dict``. Support loading
    groundtruth image from file backend.
    Args:
        image_list (list): A list of image file.
        args (CArgs): A global object setting in
            obj:``cityscapesscripts.evaluation.
            evalInstanceLevelSemanticLabeling``
        backend_args (dict, optional): Arguments to instantiate the
            preifx of uri corresponding backend. Defaults to None.
    Returns:
        dict: The processed groundtruth results.
    """
    if not HAS_CITYSCAPESAPI:
        raise RuntimeError('Failed to import `cityscapesscripts`.'
                           'Please try to install official '
                           'cityscapesscripts by '
                           '"pip install cityscapesscripts"')
    imgCount = 0
    instanceDict = {}

    if not isinstance(image_list, list):
        image_list = [image_list]

    if not args.quiet:
        print(f'Processing {len(image_list)} images...')

    for image_name in image_list:
        # Load image
        img_bytes = get(image_name, backend_args=backend_args)
        imgNp = mmcv.imfrombytes(img_bytes, flag='unchanged', backend='pillow')

        # Initialize label categories
        instances: dict = {}
        for label in labels:
            instances[label.name] = []

        # Loop through all instance ids in instance image
        for instanceId in np.unique(imgNp):
            instanceObj = Instance(imgNp, instanceId)

            instances[id2label[instanceObj.labelID].name].append(
                instanceObj.toDict())

        instanceDict[image_name] = instances
        imgCount += 1

        if not args.quiet:
            print(f'\rImages Processed: {imgCount}', end=' ')
            sys.stdout.flush()

    return instanceDict