tensorpack / tensorpack

BOX AP AP50 AP75 APS APM APL
SPEED
MODEL CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
CODE PAPER
ε-REPR
PAPER
GLOBAL RANK
Mask R-CNN
(ResNet-101-FPN, GN, Cascade)
0.477 0.474 0.661 -- 0.521 -- 0.304 -- 0.507 -- 0.615 -- 2.4 #7
Mask R-CNN
(ResNet-50-FPN, 2x)
0.389 -- 0.597 -- 0.421 -- 0.219 -- 0.426 -- 0.511 -- 2.8 #17
Mask R-CNN
(ResNet-50-FPN, GroupNorm)
0.404 0.403 0.609 0.610 0.440 0.440 0.240 0.357 0.433 0.579 0.538 0.377
2.6 #12
See Full Build Details +get badge code
[![SotaBench](https://img.shields.io/endpoint.svg?url=https://sotabench.com/api/v0/badge/gh/tensorpack/tensorpack)](https://sotabench.com/user/ppwwyyxx/repos/tensorpack/tensorpack)

How the Repository is Evaluated

The full sotabench.py file - source
# -*- coding: utf-8 -*-

import os
import sys
import tqdm
from contextlib import contextmanager

from tensorpack.predict import OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit
from tensorpack.utils.fs import download

from sotabencheval.utils import is_server
from sotabencheval.object_detection import COCOEvaluator

# import faster rcnn example
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "examples", "FasterRCNN"))
from config import finalize_configs, config as cfg  # noqa
from eval import predict_image  # noqa
from dataset import register_coco  # noqa
from dataset.coco import COCODetection  # noqa
from data import get_eval_dataflow  # noqa
from modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model  # noqa


if is_server():
    DATA_ROOT = "./.data/vision/"
else:  # local settings
    DATA_ROOT = os.path.expanduser("~/data/")
COCO_ROOT = os.path.join(DATA_ROOT, "coco")


register_coco(COCO_ROOT)


@contextmanager
def backup_cfg():
    orig_config = cfg.to_dict()
    yield
    cfg.from_dict(orig_config)


def evaluate_rcnn(model_name, paper_arxiv_id, cfg_list, model_file):
    evaluator = COCOEvaluator(
        root=COCO_ROOT, model_name=model_name, paper_arxiv_id=paper_arxiv_id
    )
    category_id_to_coco_id = {
        v: k for k, v in COCODetection.COCO_id_to_category_id.items()
    }

    cfg.update_args(cfg_list)  # TODO backup/restore config
    finalize_configs(False)
    MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(model_file),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1],
    )
    predictor = OfflinePredictor(predcfg)

    def xyxy_to_xywh(box):
        box[2] -= box[0]
        box[3] -= box[1]
        return box

    df = get_eval_dataflow("coco_val2017")
    df.reset_state()
    for img, img_id in tqdm.tqdm(df, total=len(df)):
        results = predict_image(img, predictor)
        res = [
            {
                "image_id": img_id,
                "category_id": category_id_to_coco_id.get(
                    int(r.class_id), int(r.class_id)
                ),
                "bbox": xyxy_to_xywh([round(float(x), 4) for x in r.box]),
                "score": round(float(r.score), 3),
            }
            for r in results
        ]
        evaluator.add(res)
        if evaluator.cache_exists:
            break

    evaluator.save()


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R50FPN2x.npz",
    "./",
    expect_size=165362754)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-50-FPN, 2x)", "1703.06870", [],
        "COCO-MaskRCNN-R50FPN2x.npz",
    )


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R50FPN2xGN.npz",
    "./",
    expect_size=167363872)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-50-FPN, GroupNorm)", "1803.08494",
        """FPN.NORM=GN BACKBONE.NORM=GN
FPN.FRCNN_HEAD_FUNC=fastrcnn_4conv1fc_gn_head
FPN.MRCNN_HEAD_FUNC=maskrcnn_up4conv_gn_head""".split(),
        "COCO-MaskRCNN-R50FPN2xGN.npz",
    )


download(
    "http://models.tensorpack.com/FasterRCNN/COCO-MaskRCNN-R101FPN9xGNCasAugScratch.npz",
    "./",
    expect_size=355680386)
with backup_cfg():
    evaluate_rcnn(
        "Mask R-CNN (ResNet-101-FPN, GN, Cascade)", "1811.08883",
        """
    FPN.CASCADE=True BACKBONE.RESNET_NUM_BLOCKS=[3,4,23,3] FPN.NORM=GN
    BACKBONE.NORM=GN FPN.FRCNN_HEAD_FUNC=fastrcnn_4conv1fc_gn_head
    FPN.MRCNN_HEAD_FUNC=maskrcnn_up4conv_gn_head""".split(),
        "COCO-MaskRCNN-R101FPN9xGNCasAugScratch.npz",
    )
STATUS
BUILD
COMMIT MESSAGE
RUN TIME
skip gradient packer for grads with undefined shape
ppwwyyxx   22f37f9  ·  2 days ago
unknown
ignore saver error
ppwwyyxx   0e69750  ·  Oct 13 2020
0h:07m:38s
skip creating a unused saver
ppwwyyxx   2edcf10  ·  Oct 02 2020
0h:09m:08s
update
ppwwyyxx   58fc1de  ·  Sep 30 2020
0h:09m:14s
Improve compatibility to tensorflow 2.3 (#1487) * fix gfile not…
ppwwyyxx   1b98fe5  ·  Sep 30 2020
0h:07m:10s
remove extra kwargs in TrainConfig
ppwwyyxx   a12872d  ·  Sep 03 2020
0h:09m:47s
update docs
ppwwyyxx   229e991  ·  Sep 02 2020
0h:04m:03s
update cifar-resnet
ppwwyyxx   63f656c  ·  Sep 02 2020
0h:09m:26s
update cifar-resnet
ppwwyyxx   ef0ecb8  ·  Sep 01 2020
0h:09m:36s
Add TinyImageNet
ppwwyyxx   52bbe70  ·  Aug 24 2020
0h:08m:00s
make add_moving_summary use local variables, so they are not bro…
ppwwyyxx   02e53f7  ·  Aug 24 2020
unknown
update docs
ppwwyyxx   2d661d6  ·  Aug 20 2020
unknown
add tfutils.collect_env
ppwwyyxx   379e9a0  ·  Aug 08 2020
unknown
`load_checkpoint_vars` supports npz
ppwwyyxx   57f542d  ·  Aug 08 2020
unknown
print TF build info
ppwwyyxx   43a44c1  ·  Aug 08 2020
17 days, 2h:40m:21s
fix docs
ppwwyyxx   775aa3c  ·  Jul 31 2020
0h:02m:24s
fix docs
ppwwyyxx   8774fd7  ·  Jul 31 2020
0h:04m:06s
get_all_checkpoints
ppwwyyxx   73baa9a  ·  Jul 31 2020
0h:06m:48s
tensorpack.tfv1
ppwwyyxx   425f9d2  ·  Jul 26 2020
0h:07m:43s
lint & TF2
ppwwyyxx   8e6f930  ·  Jul 26 2020
0h:07m:13s
fix usage of collective_ops.nccl
ppwwyyxx   fa0b0ca  ·  Jul 22 2020
0h:06m:51s
rewrite allreduce and avoid bug in TF's nccl
ppwwyyxx   6151e04  ·  Jul 19 2020
0h:08m:06s
CheckNumerics callbacks
ppwwyyxx   dbc0b36  ·  Jul 16 2020
0h:08m:18s
update about NVML and environment
ppwwyyxx   07e28ee  ·  Jul 15 2020
0h:07m:06s
v1 compat in optimizers
ppwwyyxx   62ea40c  ·  Jul 13 2020
0h:07m:07s
cache CI depednencies
ppwwyyxx   7dacad0  ·  Jul 11 2020
0h:08m:39s
tensorflow install in CI
ppwwyyxx   4114ed8  ·  Jul 11 2020
0h:03m:06s
upgrade detection / some v2 compat fix
ppwwyyxx   edb7897  ·  Jul 11 2020
0h:08m:09s
TF2 compat
ppwwyyxx   55b2e0f  ·  Jul 11 2020
0h:07m:42s
explain LR_SCHEDULE in configs (#1452)
ppwwyyxx   9c1b1b7  ·  Jun 04 2020
0h:08m:18s
[FasterRCNN] add clarification of data format
ppwwyyxx   b434887  ·  Jun 04 2020
0h:07m:54s
fix serving export (fix #1449)
ppwwyyxx   7c76e76  ·  Jun 01 2020
0h:08m:18s
warning about fork
ppwwyyxx d95bf7a    315eab9  ·  May 22 2020
0h:08m:25s
fix for latest flake8
ppwwyyxx   ea52556  ·  May 14 2020
0h:03m:46s
freeze_variable: don't add to collection if not originally train…
ppwwyyxx   610ffe3  ·  May 14 2020
0h:07m:52s
allow training tower with index=0 and reuse
ppwwyyxx   9f4154e  ·  May 07 2020
0h:07m:38s
update docs
ppwwyyxx   17c5984  ·  Apr 30 2020
0h:08m:29s
bump version
ppwwyyxx   8f83134  ·  Apr 24 2020
0h:03m:50s
fix lint
ppwwyyxx   d8b2990  ·  Apr 24 2020
0h:07m:42s
[imgaug] improve grayscale and gaussianblur
ppwwyyxx aa7e18f    e663713  ·  Apr 24 2020
0h:08m:17s
update example
ppwwyyxx   acb441c  ·  Apr 19 2020
0h:08m:46s
[a3c] specify dir to save train logs
ppwwyyxx   21a6984  ·  Apr 16 2020
0h:09m:06s
Prevent filtering of empty images (#1424) * Update data.py *…
ppwwyyxx   963e510  ·  Apr 15 2020
0h:08m:22s
update docs
ppwwyyxx   92aab65  ·  Apr 12 2020
0h:07m:30s
fix docs/conf.py
ppwwyyxx   9460065  ·  Apr 11 2020
0h:03m:09s
update sphinx==3.0
ppwwyyxx   399a74d  ·  Apr 11 2020
0h:08m:27s
windows support for Image2Image (fix #1413)
ppwwyyxx   031e698  ·  Mar 27 2020
0h:08m:12s
make Image2Image support windows (fix #1412)
ppwwyyxx   f1a8acf  ·  Mar 26 2020
0h:03m:46s
update
ppwwyyxx   e50423d  ·  Mar 26 2020
0h:08m:09s
bump version
ppwwyyxx   7958d60  ·  Mar 23 2020
0h:07m:37s
update docs about MapAndBatch
ppwwyyxx   de9025d  ·  Mar 23 2020
0h:08m:51s
update
ppwwyyxx   22582cc  ·  Mar 17 2020
0h:07m:18s
fix syncbn
ppwwyyxx   bc5b5cc  ·  Mar 17 2020
0h:07m:17s
use TowerHandle.get_tensor to access variables (fix #1409)
ppwwyyxx   d89b6f0  ·  Mar 14 2020
0h:08m:44s
fix import
ppwwyyxx   6f0ba59  ·  Mar 11 2020
0h:03m:48s
error for wrong param dtype
ppwwyyxx   b01cd6d  ·  Mar 11 2020
unknown
cleanup deprecations
ppwwyyxx e0a7e8f    e2eba74  ·  Mar 11 2020
0h:07m:10s
Convert zmq input to dataset
ppwwyyxx   9f4600a  ·  Mar 11 2020
0h:08m:06s
update
ppwwyyxx   c2d99a4  ·  Mar 06 2020
0h:07m:20s
reduce memory of MapAndBatch
ppwwyyxx   b2d106a  ·  Mar 06 2020
unknown
speed up dataloader: casting & clip
ppwwyyxx   2ff9a5f  ·  Mar 08 2020
unknown
standarize arg names in LayerNorm/InstanceNorm
ppwwyyxx   07e464d  ·  Mar 08 2020
unknown
fix typo (#1403)
ppwwyyxx   f128a5c  ·  Feb 27 2020
0h:08m:54s
fix #1402 Signed-off-by: Yuxin Wu <[email protected]>
ppwwyyxx   83e00d7  ·  Feb 25 2020
0h:07m:23s
bump version
ppwwyyxx   3445a76  ·  Feb 22 2020
0h:06m:40s
update
ppwwyyxx   b28cfa8  ·  Feb 17 2020
0h:08m:39s
fix input dtype not matching variable dtype (#1386) * fix not i…
ppwwyyxx   4ac2e22  ·  Jan 22 2020
0h:07m:45s
Fix Faster-RCNN d2_convert error. (#1389)
ppwwyyxx   12ad257  ·  Jan 22 2020
0h:07m:43s
Update issue templates
ppwwyyxx   882f858  ·  Jan 13 2020
0h:07m:10s
update docs formatting
ppwwyyxx   b88a40a  ·  Jan 13 2020
0h:07m:32s
update docs
ppwwyyxx   8cfcc47  ·  Jan 13 2020
0h:07m:26s
fix bug in 552c2b3b51c83fc27bdf6df691f1d707e40865f9
ppwwyyxx   3c14ff6  ·  Jan 11 2020
0h:06m:52s
fix LMDBSerializer
ppwwyyxx   ffd9eae  ·  Jan 09 2020
0h:07m:44s
lint with flake8-bugbear
ppwwyyxx   bf8acbf  ·  Jan 09 2020
0h:07m:15s
lint with flake8-comprehensions
ppwwyyxx   552c2b3  ·  Jan 09 2020
0h:02m:49s
fix bug when combining DataParallelInferenceRunner+BatchNorm (si…
ppwwyyxx   a995070  ·  Jan 09 2020
0h:06m:57s
correct TF version in saliency example
ppwwyyxx   d2f9564  ·  Jan 07 2020
0h:07m:09s
mention imagenet synsets.
ppwwyyxx   758ae94  ·  Jan 03 2020
0h:06m:38s
more pickability in dataflow
ppwwyyxx   4768b26  ·  Dec 27 2019
0h:02m:36s
Switch to pickle by default. In Python3 pickle is as fast as msg…
ppwwyyxx   1648ddd  ·  Dec 27 2019
0h:06m:38s
fix output handling of subproc_call
ppwwyyxx   34533d6  ·  Dec 19 2019
0h:10m:52s
point8->point4
ppwwyyxx   48546d5  ·  Dec 18 2019
0h:09m:28s
use yield from in dataflow; update logger name in dataflow.
ppwwyyxx   9fac1a6  ·  Nov 26 2019
unknown
Drop Python2 support
ppwwyyxx   0641618  ·  Nov 10 2019
unknown
add serialization benchmark & forking pickler
ppwwyyxx   6166340  ·  Nov 10 2019
unknown
unknown
update example d2 conversion command. (fix #1356)
ppwwyyxx   23ab700  ·  Oct 29 2019
unknown
fix lint
ppwwyyxx   7e923bf  ·  Oct 19 2019
unknown
fix the use of cfg in workers
ppwwyyxx   74badc6  ·  Oct 19 2019
unknown
update benchmarks with D2
ppwwyyxx   a6ca79c  ·  Oct 10 2019
unknown
add D2 conversion
ppwwyyxx   53b887f  ·  Oct 10 2019
unknown
update tp data dir
ppwwyyxx   a7d6fd1  ·  Oct 07 2019
unknown
CI: create data dir
ppwwyyxx   6c26e32  ·  Oct 07 2019
unknown
CI: use separate data dir for each job. hopefully makes it less …
ppwwyyxx   226016f  ·  Oct 07 2019
unknown
Fixed some deprecation warnings on python 3 (#1337)
ppwwyyxx   7731d89  ·  Oct 07 2019
unknown
unknown
assert class ids not out of bounds (#1336)
ppwwyyxx   bbf29a1  ·  Oct 06 2019
unknown
update docs
ppwwyyxx   17cb355  ·  Oct 06 2019
unknown
update docs and sotabench
ppwwyyxx   e83b079  ·  Oct 06 2019
unknown
re-benchmark Mask R-CNN
ppwwyyxx   7c1c987  ·  Oct 04 2019
unknown
# 9
fix import path
ppwwyyxx   2014358  ·  Oct 03 2019
unknown
# 8
update sotabench config
ppwwyyxx   202cfb6  ·  Oct 03 2019
unknown
# 7
[horovod] no need to broadcast every epoch
ppwwyyxx   caafda8  ·  Oct 03 2019
unknown
# 6
Make disable_layer_logging public
ppwwyyxx   e53bf22  ·  Oct 01 2019
unknown
# 5
[MaskRCNN] add a better R50
ppwwyyxx   cc2322b  ·  Sep 30 2019
unknown
# 4
sotabench: install missing deps
ppwwyyxx   52795cc  ·  Sep 29 2019
unknown
# 3
sotabench: install missing deps
ppwwyyxx   d52d288  ·  Sep 29 2019
unknown
# 2
update sotabench setup script
ppwwyyxx   171f124  ·  Sep 29 2019
unknown
# 1
add sotabench
ppwwyyxx   a414092  ·  Sep 29 2019
unknown
# 0
unknown