TEST PERPLEXITY
SPEED
MODEL CODE PAPER
ε-REPR
PAPER
GLOBAL RANK
Transformer-XL
(RMS dynamic eval)
16.44 16.40 483.6 #2
See Full Build Details +get badge code
[![SotaBench](https://img.shields.io/endpoint.svg?url=https://sotabench.com/api/v0/badge/gh/PiotrCzapla/dynamiceval-transformer)](https://sotabench.com/user/piotr.czapla-priv/repos/PiotrCzapla/dynamiceval-transformer)

How the Repository is Evaluated

The full sotabench.py file - source

from absl import flags
from absl import app
import sys
import os

from sotabencheval.language_modelling import WikiText103Evaluator, WikiText2Evaluator
import tensorflow as tf


dir_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(f"{dir_path}/tf")  # add tf. to path to import dynamic_eval
#os.chdir(f"{dir_path}/tf")
from dynamiceval_tf_copy_for_sotabench import dynamic_eval, data_utils, FLAGS
#os.chdir(f"{dir_path}")

def main(unused_argv=None):
    print("unused_argv", unused_argv)
    tf.logging.set_verbosity(tf.logging.INFO)
    # Get corpus info
    corpus_info = data_utils.get_corpus_info(FLAGS.corpus_info_path)
    n_token = corpus_info["vocab_size"]
    cutoffs = corpus_info["cutoffs"][1:-1]
    tf.logging.info("n_token {}".format(n_token))

    evaluator = WikiText103Evaluator(
        model_name="Transformer-XL (RMS dynamic eval)",
        paper_arxiv_id="1904.08378",
        paper_pwc_id="dynamic-evaluation-of-transformer-language",
        #expected perplexity: 16.40
    
    ).eval(dynamic_eval(n_token, cutoffs, "/gpu:0"))
    # test why the results are not being saved, temporary mocking the evaluation
    # import numpy as np
    # evaluator._neglogloss =  np.log(16.441230035231136) * evaluator.dataset.testset_size
    evaluator.print_stats()
    evaluator.print_results()
    evaluator.save()  # to double check that it is being called

if __name__ == "__main__":
    import sys
    argv = f"""
    IGNORED-PROGNAME
        --data_dir={dir_path}/tf/pretrained_xl/tf_wt103/data/tfrecords
        --record_info_dir={dir_path}/tf/pretrained_xl/tf_wt103/data/tfrecords/
        --corpus_info_path={dir_path}/tf/pretrained_xl/tf_wt103/data/corpus-info.json
        --eval_ckpt_path={dir_path}/tf/pretrained_xl/tf_wt103/model/model.ckpt-0
        --model_dir=EXP-wt103
        --div_val=4
        --learning_rate=0.000002
        --decay_rate=0
        --epsilon=0.00001
        --rms=True
        --untie_r=True
        --proj_share_all_but_first=True
        --num_core_per_host=1
        --n_layer=18
        --d_model=1024
        --d_embed=1024
        --n_head=16
        --d_head=64
        --d_inner=4096
        --dropout=0.0
        --dropatt=0.0
        --tgt_len=128
        --mem_len=1600
        --clamp_len=1000
        --eval_split=test
        --same_length=True
    """.split()

    FLAGS(argv, known_only=True)
    assert FLAGS.data_dir == f"{dir_path}/tf/pretrained_xl/tf_wt103/data/tfrecords"
    main()
STATUS
BUILD
COMMIT MESSAGE
RUN TIME
Remove chdir it shouldn't be needed.
PiotrCzapla   010e058  ·  Oct 09 2019
0h:15m:26s
0h:12m:34s
Fake execution to test saving.
PiotrCzapla   9a58bc2  ·  Oct 08 2019
0h:11m:14s
0h:16m:28s
0h:17m:14s
add one more call to .save
PiotrCzapla   5e1949a  ·  Oct 07 2019
0h:11m:24s
Style change to trigger a build
PiotrCzapla   f4a420f  ·  Oct 08 2019
0h:18m:30s
Finall fixing after testing in docker container
PiotrCzapla   43cbb0c  ·  Oct 07 2019
0h:20m:33s
Fix pip2.7 installation
PiotrCzapla   8296ba0  ·  Oct 07 2019
0h:06m:48s
install missing pip2.7
PiotrCzapla   72dc943  ·  Oct 07 2019
0h:04m:05s
fix loss (it was still averaged)
PiotrCzapla   307cd04  ·  Oct 07 2019
0h:04m:24s
install python 2.7
PiotrCzapla   29d0d86  ·  Oct 07 2019
0h:04m:58s
Fix loss reporting
PiotrCzapla   12eb1ce  ·  Oct 07 2019
0h:04m:45s
Fix Flag parsing
PiotrCzapla   47b0224  ·  Oct 07 2019
0h:04m:57s
Fix model download
PiotrCzapla   a173618  ·  Oct 07 2019
0h:04m:10s
sotabench: Fix model downloading
PiotrCzapla   a1f1220  ·  Oct 07 2019
0h:04m:14s
Use cuda 10
PiotrCzapla   4e55842  ·  Oct 07 2019
0h:04m:45s
0h:04m:39s