tensorpack / tensorpack

BOX AP AP50 AP75 APS APM APL
SPEED
MODEL CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
PAPER
GLOBAL RANK
Mask R-CNN
(ResNet-101-FPN, GN, Cascade)
0.477 0.474 0.661 -- 0.521 -- 0.304 -- 0.507 -- 0.615 -- 2.4 #6
Mask R-CNN
(ResNet-50-FPN, 2x)
0.389 -- 0.597 -- 0.421 -- 0.219 -- 0.426 -- 0.511 -- 2.8 #80
Mask R-CNN
(ResNet-50-FPN, GroupNorm)
0.404 0.403 0.609 0.610 0.440 0.440 0.240 0.357 0.433 0.579 0.538 0.377
2.6 #57
See Full Build Details +get badge code
[![SotaBench](https://img.shields.io/endpoint.svg?url=https://sotabench.com/api/v0/badge/gh/tensorpack/tensorpack)](https://sotabench.com/user/ppwwyyxx/repos/tensorpack/tensorpack)

How the Repository is Evaluated

The full sotabench.py file - source
# -*- coding: utf-8 -*-

import os
import sys
import tqdm
from contextlib import contextmanager

from tensorpack.predict import OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit
from tensorpack.utils.fs import download

from sotabencheval.utils import is_server
from sotabencheval.object_detection import COCOEvaluator

# import faster rcnn example
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "examples", "FasterRCNN"))
from config import finalize_configs, config as cfg  # noqa
from eval import predict_image  # noqa
from dataset import register_coco  # noqa
from dataset.coco import COCODetection  # noqa
from data import get_eval_dataflow  # noqa
from modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model  # noqa


if is_server():
    DATA_ROOT = "./.data/vision/"
else:  # local settings
    DATA_ROOT = os.path.expanduser("~/data/")
COCO_ROOT = os.path.join(DATA_ROOT, "coco")


register_coco(COCO_ROOT)


@contextmanager
def backup_cfg():
    orig_config = cfg.to_dict()
    yield
    cfg.from_dict(orig_config)


def evaluate_rcnn(model_name, paper_arxiv_id, cfg_list, model_file):
    evaluator = COCOEvaluator(
        root=COCO_ROOT, model_name=model_name, paper_arxiv_id=paper_arxiv_id
    )
    category_id_to_coco_id = {
        v: k for k, v in COCODetection.COCO_id_to_category_id.items()
    }

    cfg.update_args(cfg_list)  # TODO backup/restore config
    finalize_configs(False)
    MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(model_file),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1],
    )
    predictor = OfflinePredictor(predcfg)

    def xyxy_to_xywh(box):
        box[2] -= box[0]
        box[3] -= box[1]
        return box

    df = get_eval_dataflow("coco_val2017")
    df.reset_state()
    for img, img_id in tqdm.tqdm(df, total=len(df)):
        results = predict_image(img, predictor)
        res = [
            {
                "image_id": img_id,
                "category_id": category_id_to_coco_id.get(
                    int(r.class_id), int(r.class_id)
                ),
                "bbox": xyxy_to_xywh([round(float(x), 4) for x in r.box]),
                "score": round(float(r.score), 3),
            }
            for r in results
        ]
        evaluator.add(res)
        if evaluator.cache_exists:
            break

    evaluator.save()


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R50FPN2x.npz",
    "./",
    expect_size=165362754)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-50-FPN, 2x)", "1703.06870", [],
        "COCO-MaskRCNN-R50FPN2x.npz",
    )


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R50FPN2xGN.npz",
    "./",
    expect_size=167363872)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-50-FPN, GroupNorm)", "1803.08494",
        """FPN.NORM=GN BACKBONE.NORM=GN
FPN.FRCNN_HEAD_FUNC=fastrcnn_4conv1fc_gn_head
FPN.MRCNN_HEAD_FUNC=maskrcnn_up4conv_gn_head""".split(),
        "COCO-MaskRCNN-R50FPN2xGN.npz",
    )


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R101FPN9xGNCasAugScratch.npz",
    "./",
    expect_size=355680386)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-101-FPN, GN, Cascade)", "1811.08883",
        """
    FPN.CASCADE=True BACKBONE.RESNET_NUM_BLOCKS=[3,4,23,3] FPN.NORM=GN
    BACKBONE.NORM=GN FPN.FRCNN_HEAD_FUNC=fastrcnn_4conv1fc_gn_head
    FPN.MRCNN_HEAD_FUNC=maskrcnn_up4conv_gn_head""".split(),
        "COCO-MaskRCNN-R101FPN9xGNCasAugScratch.npz",
    )
STATUS
BUILD
COMMIT MESSAGE
RUN TIME
use yield from in dataflow; update logger name in dataflow.
ppwwyyxx   9fac1a6  ·  Nov 26 2019
0h:09m:34s
Drop Python2 support
ppwwyyxx   0641618  ·  Nov 10 2019
0h:09m:30s
add serialization benchmark & forking pickler
ppwwyyxx   6166340  ·  Nov 10 2019
0h:09m:35s
0h:04m:54s
update example d2 conversion command. (fix #1356)
ppwwyyxx   23ab700  ·  Oct 29 2019
0h:05m:01s
fix lint
ppwwyyxx   7e923bf  ·  Oct 19 2019
0h:10m:47s
fix the use of cfg in workers
ppwwyyxx   74badc6  ·  Oct 19 2019
0h:10m:07s
update benchmarks with D2
ppwwyyxx   a6ca79c  ·  Oct 10 2019
0h:08m:58s
add D2 conversion
ppwwyyxx   53b887f  ·  Oct 10 2019
0h:09m:01s
update tp data dir
ppwwyyxx   a7d6fd1  ·  Oct 07 2019
1h:43m:54s
CI: create data dir
ppwwyyxx   6c26e32  ·  Oct 07 2019
0h:10m:51s
CI: use separate data dir for each job. hopefully makes it less …
ppwwyyxx   226016f  ·  Oct 07 2019
0h:35m:32s
Fixed some deprecation warnings on python 3 (#1337)
ppwwyyxx   7731d89  ·  Oct 07 2019
unknown
0h:07m:57s
assert class ids not out of bounds (#1336)
ppwwyyxx   bbf29a1  ·  Oct 06 2019
0h:08m:46s
update docs
ppwwyyxx   17cb355  ·  Oct 06 2019
0h:08m:36s
update docs and sotabench
ppwwyyxx   e83b079  ·  Oct 06 2019
1h:07m:59s
re-benchmark Mask R-CNN
ppwwyyxx   7c1c987  ·  Oct 04 2019
0h:07m:10s
fix import path
ppwwyyxx   2014358  ·  Oct 03 2019
0h:41m:05s
update sotabench config
ppwwyyxx   202cfb6  ·  Oct 03 2019
0h:05m:07s
[horovod] no need to broadcast every epoch
ppwwyyxx   caafda8  ·  Oct 03 2019
0h:06m:16s
Make disable_layer_logging public
ppwwyyxx   e53bf22  ·  Oct 01 2019
0h:07m:25s
[MaskRCNN] add a better R50
ppwwyyxx   cc2322b  ·  Sep 30 2019
0h:06m:12s
sotabench: install missing deps
ppwwyyxx   52795cc  ·  Sep 29 2019
0h:06m:15s
sotabench: install missing deps
ppwwyyxx   d52d288  ·  Sep 29 2019
0h:04m:42s
update sotabench setup script
ppwwyyxx   171f124  ·  Sep 29 2019
0h:04m:30s
add sotabench
ppwwyyxx   a414092  ·  Sep 29 2019
0h:03m:25s
0h:03m:43s