Update docs 11/20/2021

2026-06-16 12:44:41 +00:00 · 2021-11-20 20:47:58 -08:00
parent f2d2a25aff
commit a4cd5e851a
32 changed files with 7131 additions and 0 deletions
@@ -0,0 +1,85 @@
+ml.rl.evaluation package
+========================
+
+Submodules
+----------
+
+ml.rl.evaluation.cpe module
+---------------------------
+
+.. automodule:: ml.rl.evaluation.cpe
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.doubly\_robust\_estimator module
+-------------------------------------------------
+
+.. automodule:: ml.rl.evaluation.doubly_robust_estimator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.evaluation\_data\_page module
+----------------------------------------------
+
+.. automodule:: ml.rl.evaluation.evaluation_data_page
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.evaluator module
+---------------------------------
+
+.. automodule:: ml.rl.evaluation.evaluator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.ranking\_evaluator module
+------------------------------------------
+
+.. automodule:: ml.rl.evaluation.ranking_evaluator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.reward\_net\_evaluator module
+----------------------------------------------
+
+.. automodule:: ml.rl.evaluation.reward_net_evaluator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.sequential\_doubly\_robust\_estimator module
+-------------------------------------------------------------
+
+.. automodule:: ml.rl.evaluation.sequential_doubly_robust_estimator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.weighted\_sequential\_doubly\_robust\_estimator module
+-----------------------------------------------------------------------
+
+.. automodule:: ml.rl.evaluation.weighted_sequential_doubly_robust_estimator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.evaluation.world\_model\_evaluator module
+-----------------------------------------------
+
+.. automodule:: ml.rl.evaluation.world_model_evaluator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.evaluation
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,157 @@
+ml.rl.models package
+====================
+
+Submodules
+----------
+
+ml.rl.models.actor module
+-------------------------
+
+.. automodule:: ml.rl.models.actor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.base module
+------------------------
+
+.. automodule:: ml.rl.models.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.bcq module
+-----------------------
+
+.. automodule:: ml.rl.models.bcq
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.categorical\_dqn module
+------------------------------------
+
+.. automodule:: ml.rl.models.categorical_dqn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.cem\_planner module
+--------------------------------
+
+.. automodule:: ml.rl.models.cem_planner
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.convolutional\_network module
+------------------------------------------
+
+.. automodule:: ml.rl.models.convolutional_network
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.dqn module
+-----------------------
+
+.. automodule:: ml.rl.models.dqn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.dueling\_q\_network module
+---------------------------------------
+
+.. automodule:: ml.rl.models.dueling_q_network
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.dueling\_quantile\_dqn module
+------------------------------------------
+
+.. automodule:: ml.rl.models.dueling_quantile_dqn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.example\_sequence\_model module
+--------------------------------------------
+
+.. automodule:: ml.rl.models.example_sequence_model
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.fully\_connected\_network module
+---------------------------------------------
+
+.. automodule:: ml.rl.models.fully_connected_network
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.mdn\_rnn module
+----------------------------
+
+.. automodule:: ml.rl.models.mdn_rnn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.no\_soft\_update\_embedding module
+-----------------------------------------------
+
+.. automodule:: ml.rl.models.no_soft_update_embedding
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.parametric\_dqn module
+-----------------------------------
+
+.. automodule:: ml.rl.models.parametric_dqn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.quantile\_dqn module
+---------------------------------
+
+.. automodule:: ml.rl.models.quantile_dqn
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.seq2slate module
+-----------------------------
+
+.. automodule:: ml.rl.models.seq2slate
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.seq2slate\_reward module
+-------------------------------------
+
+.. automodule:: ml.rl.models.seq2slate_reward
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.models.world\_model module
+--------------------------------
+
+.. automodule:: ml.rl.models.world_model
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.models
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,37 @@
+ml.rl.polyfill package
+======================
+
+Submodules
+----------
+
+ml.rl.polyfill.decorators module
+--------------------------------
+
+.. automodule:: ml.rl.polyfill.decorators
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.polyfill.exceptions module
+--------------------------------
+
+.. automodule:: ml.rl.polyfill.exceptions
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.polyfill.types module
+---------------------------
+
+.. automodule:: ml.rl.polyfill.types
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.polyfill
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,29 @@
+ml.rl.prediction package
+========================
+
+Submodules
+----------
+
+ml.rl.prediction.dqn\_torch\_predictor module
+---------------------------------------------
+
+.. automodule:: ml.rl.prediction.dqn_torch_predictor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.prediction.predictor\_wrapper module
+------------------------------------------
+
+.. automodule:: ml.rl.prediction.predictor_wrapper
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.prediction
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,61 @@
+ml.rl.preprocessing package
+===========================
+
+Submodules
+----------
+
+ml.rl.preprocessing.batch\_preprocessor module
+----------------------------------------------
+
+.. automodule:: ml.rl.preprocessing.batch_preprocessor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.preprocessing.identify\_types module
+------------------------------------------
+
+.. automodule:: ml.rl.preprocessing.identify_types
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.preprocessing.normalization module
+----------------------------------------
+
+.. automodule:: ml.rl.preprocessing.normalization
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.preprocessing.postprocessor module
+----------------------------------------
+
+.. automodule:: ml.rl.preprocessing.postprocessor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.preprocessing.preprocessor module
+---------------------------------------
+
+.. automodule:: ml.rl.preprocessing.preprocessor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.preprocessing.sparse\_to\_dense module
+--------------------------------------------
+
+.. automodule:: ml.rl.preprocessing.sparse_to_dense
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.preprocessing
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,45 @@
+ml.rl.readers package
+=====================
+
+Submodules
+----------
+
+ml.rl.readers.base module
+-------------------------
+
+.. automodule:: ml.rl.readers.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.readers.data\_streamer module
+-----------------------------------
+
+.. automodule:: ml.rl.readers.data_streamer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.readers.json\_dataset\_reader module
+------------------------------------------
+
+.. automodule:: ml.rl.readers.json_dataset_reader
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.readers.nparray\_reader module
+------------------------------------
+
+.. automodule:: ml.rl.readers.nparray_reader
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.readers
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,85 @@
+ml.rl package
+=============
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   ml.rl.evaluation
+   ml.rl.models
+   ml.rl.polyfill
+   ml.rl.prediction
+   ml.rl.preprocessing
+   ml.rl.readers
+   ml.rl.simulators
+   ml.rl.training
+   ml.rl.workflow
+
+Submodules
+----------
+
+ml.rl.caffe\_utils module
+-------------------------
+
+.. automodule:: ml.rl.caffe_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.debug\_on\_error module
+-----------------------------
+
+.. automodule:: ml.rl.debug_on_error
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.json\_serialize module
+----------------------------
+
+.. automodule:: ml.rl.json_serialize
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.parameters module
+-----------------------
+
+.. automodule:: ml.rl.parameters
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.tensorboardX module
+-------------------------
+
+.. automodule:: ml.rl.tensorboardX
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.torch\_utils module
+-------------------------
+
+.. automodule:: ml.rl.torch_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.types module
+------------------
+
+.. automodule:: ml.rl.types
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,21 @@
+ml.rl.simulators package
+========================
+
+Submodules
+----------
+
+ml.rl.simulators.recsim module
+------------------------------
+
+.. automodule:: ml.rl.simulators.recsim
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.simulators
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,29 @@
+ml.rl.training.gradient\_free package
+=====================================
+
+Submodules
+----------
+
+ml.rl.training.gradient\_free.es\_worker module
+-----------------------------------------------
+
+.. automodule:: ml.rl.training.gradient_free.es_worker
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.gradient\_free.evolution\_pool module
+----------------------------------------------------
+
+.. automodule:: ml.rl.training.gradient_free.evolution_pool
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.training.gradient_free
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,29 @@
+ml.rl.training.ranking package
+==============================
+
+Submodules
+----------
+
+ml.rl.training.ranking.seq2slate\_tf\_trainer module
+----------------------------------------------------
+
+.. automodule:: ml.rl.training.ranking.seq2slate_tf_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.ranking.seq2slate\_trainer module
+------------------------------------------------
+
+.. automodule:: ml.rl.training.ranking.seq2slate_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.training.ranking
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,159 @@
+ml.rl.training package
+======================
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   ml.rl.training.gradient_free
+   ml.rl.training.ranking
+   ml.rl.training.world_model
+
+Submodules
+----------
+
+ml.rl.training.c51\_trainer module
+----------------------------------
+
+.. automodule:: ml.rl.training.c51_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.cem\_trainer module
+----------------------------------
+
+.. automodule:: ml.rl.training.cem_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.dqn\_trainer module
+----------------------------------
+
+.. automodule:: ml.rl.training.dqn_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.dqn\_trainer\_base module
+----------------------------------------
+
+.. automodule:: ml.rl.training.dqn_trainer_base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.imitator\_training module
+----------------------------------------
+
+.. automodule:: ml.rl.training.imitator_training
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.loss\_reporter module
+------------------------------------
+
+.. automodule:: ml.rl.training.loss_reporter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.on\_policy\_predictor module
+-------------------------------------------
+
+.. automodule:: ml.rl.training.on_policy_predictor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.parametric\_dqn\_trainer module
+----------------------------------------------
+
+.. automodule:: ml.rl.training.parametric_dqn_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.qrdqn\_trainer module
+------------------------------------
+
+.. automodule:: ml.rl.training.qrdqn_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.reward\_network\_trainer module
+----------------------------------------------
+
+.. automodule:: ml.rl.training.reward_network_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.rl\_dataset module
+---------------------------------
+
+.. automodule:: ml.rl.training.rl_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.rl\_trainer\_pytorch module
+------------------------------------------
+
+.. automodule:: ml.rl.training.rl_trainer_pytorch
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.sac\_trainer module
+----------------------------------
+
+.. automodule:: ml.rl.training.sac_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.slate\_q\_trainer module
+---------------------------------------
+
+.. automodule:: ml.rl.training.slate_q_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.td3\_trainer module
+----------------------------------
+
+.. automodule:: ml.rl.training.td3_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.trainer module
+-----------------------------
+
+.. automodule:: ml.rl.training.trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.training.training\_data\_page module
+------------------------------------------
+
+.. automodule:: ml.rl.training.training_data_page
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.training
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,21 @@
+ml.rl.training.world\_model package
+===================================
+
+Submodules
+----------
+
+ml.rl.training.world\_model.mdnrnn\_trainer module
+--------------------------------------------------
+
+.. automodule:: ml.rl.training.world_model.mdnrnn_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.training.world_model
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,77 @@
+ml.rl.workflow package
+======================
+
+Submodules
+----------
+
+ml.rl.workflow.base\_workflow module
+------------------------------------
+
+.. automodule:: ml.rl.workflow.base_workflow
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.create\_normalization\_metadata module
+-----------------------------------------------------
+
+.. automodule:: ml.rl.workflow.create_normalization_metadata
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.dqn\_workflow module
+-----------------------------------
+
+.. automodule:: ml.rl.workflow.dqn_workflow
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.helpers module
+-----------------------------
+
+.. automodule:: ml.rl.workflow.helpers
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.page\_handler module
+-----------------------------------
+
+.. automodule:: ml.rl.workflow.page_handler
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.parametric\_dqn\_workflow module
+-----------------------------------------------
+
+.. automodule:: ml.rl.workflow.parametric_dqn_workflow
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.preprocess\_handler module
+-----------------------------------------
+
+.. automodule:: ml.rl.workflow.preprocess_handler
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+ml.rl.workflow.transitional module
+----------------------------------
+
+.. automodule:: ml.rl.workflow.transitional
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ml.rl.workflow
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,18 @@
+ml package
+==========
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   ml.rl
+
+Module contents
+---------------
+
+.. automodule:: ml
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,21 @@
+reagent.training.cb package
+===========================
+
+Submodules
+----------
+
+reagent.training.cb.linucb\_trainer module
+------------------------------------------
+
+.. automodule:: reagent.training.cb.linucb_trainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: reagent.training.cb
+   :members:
+   :undoc-members:
+   :show-inheritance:
@@ -0,0 +1,278 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl package" href="ml.rl.html" />
+    <link rel="prev" title="ml" href="modules.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="modules.html">All Modules</a><ul class="current">
+<li class="toctree-l2 current"><a class="current reference internal" href="#">ml package</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="#subpackages">Subpackages</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.html">ml.rl package</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#module-ml">Module contents</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="modules.html">ml</a> &raquo;</li>
+      <li>ml package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-package">
+<h1>ml package<a class="headerlink" href="#ml-package" title="Permalink to this headline"></a></h1>
+<section id="subpackages">
+<h2>Subpackages<a class="headerlink" href="#subpackages" title="Permalink to this headline"></a></h2>
+<div class="toctree-wrapper compound">
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.html">ml.rl package</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#subpackages">Subpackages</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.evaluation.html">ml.rl.evaluation package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#module-ml.rl.evaluation.cpe">ml.rl.evaluation.cpe module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-doubly-robust-estimator-module">ml.rl.evaluation.doubly_robust_estimator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-evaluation-data-page-module">ml.rl.evaluation.evaluation_data_page module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-evaluator-module">ml.rl.evaluation.evaluator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-ranking-evaluator-module">ml.rl.evaluation.ranking_evaluator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#module-ml.rl.evaluation.reward_net_evaluator">ml.rl.evaluation.reward_net_evaluator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-sequential-doubly-robust-estimator-module">ml.rl.evaluation.sequential_doubly_robust_estimator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-weighted-sequential-doubly-robust-estimator-module">ml.rl.evaluation.weighted_sequential_doubly_robust_estimator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#ml-rl-evaluation-world-model-evaluator-module">ml.rl.evaluation.world_model_evaluator module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.evaluation.html#module-ml.rl.evaluation">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.models.html">ml.rl.models package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.actor">ml.rl.models.actor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.base">ml.rl.models.base module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.bcq">ml.rl.models.bcq module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.categorical_dqn">ml.rl.models.categorical_dqn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.cem_planner">ml.rl.models.cem_planner module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.convolutional_network">ml.rl.models.convolutional_network module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.dqn">ml.rl.models.dqn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.dueling_q_network">ml.rl.models.dueling_q_network module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.dueling_quantile_dqn">ml.rl.models.dueling_quantile_dqn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.example_sequence_model">ml.rl.models.example_sequence_model module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.fully_connected_network">ml.rl.models.fully_connected_network module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.mdn_rnn">ml.rl.models.mdn_rnn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.no_soft_update_embedding">ml.rl.models.no_soft_update_embedding module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.parametric_dqn">ml.rl.models.parametric_dqn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.quantile_dqn">ml.rl.models.quantile_dqn module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.seq2slate">ml.rl.models.seq2slate module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.seq2slate_reward">ml.rl.models.seq2slate_reward module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models.world_model">ml.rl.models.world_model module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.models.html#module-ml.rl.models">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.polyfill.html">ml.rl.polyfill package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.polyfill.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.polyfill.html#module-ml.rl.polyfill.decorators">ml.rl.polyfill.decorators module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.polyfill.html#module-ml.rl.polyfill.exceptions">ml.rl.polyfill.exceptions module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.polyfill.html#module-ml.rl.polyfill.types">ml.rl.polyfill.types module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.polyfill.html#module-ml.rl.polyfill">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.prediction.html">ml.rl.prediction package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.prediction.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.prediction.html#ml-rl-prediction-dqn-torch-predictor-module">ml.rl.prediction.dqn_torch_predictor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.prediction.html#ml-rl-prediction-predictor-wrapper-module">ml.rl.prediction.predictor_wrapper module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.prediction.html#module-ml.rl.prediction">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.preprocessing.html">ml.rl.preprocessing package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#ml-rl-preprocessing-batch-preprocessor-module">ml.rl.preprocessing.batch_preprocessor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#module-ml.rl.preprocessing.identify_types">ml.rl.preprocessing.identify_types module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#ml-rl-preprocessing-normalization-module">ml.rl.preprocessing.normalization module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#ml-rl-preprocessing-postprocessor-module">ml.rl.preprocessing.postprocessor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#ml-rl-preprocessing-preprocessor-module">ml.rl.preprocessing.preprocessor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#ml-rl-preprocessing-sparse-to-dense-module">ml.rl.preprocessing.sparse_to_dense module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.preprocessing.html#module-ml.rl.preprocessing">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.readers.html">ml.rl.readers package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#module-ml.rl.readers.base">ml.rl.readers.base module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#module-ml.rl.readers.data_streamer">ml.rl.readers.data_streamer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#module-ml.rl.readers.json_dataset_reader">ml.rl.readers.json_dataset_reader module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#module-ml.rl.readers.nparray_reader">ml.rl.readers.nparray_reader module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.readers.html#module-ml.rl.readers">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.simulators.html">ml.rl.simulators package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.simulators.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.simulators.html#module-ml.rl.simulators.recsim">ml.rl.simulators.recsim module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.simulators.html#module-ml.rl.simulators">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.html">ml.rl.training package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#subpackages">Subpackages</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-c51-trainer-module">ml.rl.training.c51_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-cem-trainer-module">ml.rl.training.cem_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-module">ml.rl.training.dqn_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-base-module">ml.rl.training.dqn_trainer_base module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-imitator-training-module">ml.rl.training.imitator_training module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.loss_reporter">ml.rl.training.loss_reporter module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.on_policy_predictor">ml.rl.training.on_policy_predictor module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-parametric-dqn-trainer-module">ml.rl.training.parametric_dqn_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-qrdqn-trainer-module">ml.rl.training.qrdqn_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.reward_network_trainer">ml.rl.training.reward_network_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.rl_dataset">ml.rl.training.rl_dataset module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-rl-trainer-pytorch-module">ml.rl.training.rl_trainer_pytorch module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-sac-trainer-module">ml.rl.training.sac_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-slate-q-trainer-module">ml.rl.training.slate_q_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-td3-trainer-module">ml.rl.training.td3_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.trainer">ml.rl.training.trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.training_data_page">ml.rl.training.training_data_page module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.workflow.html">ml.rl.workflow package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-base-workflow-module">ml.rl.workflow.base_workflow module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-create-normalization-metadata-module">ml.rl.workflow.create_normalization_metadata module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-dqn-workflow-module">ml.rl.workflow.dqn_workflow module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-helpers-module">ml.rl.workflow.helpers module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-page-handler-module">ml.rl.workflow.page_handler module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-parametric-dqn-workflow-module">ml.rl.workflow.parametric_dqn_workflow module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-preprocess-handler-module">ml.rl.workflow.preprocess_handler module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#ml-rl-workflow-transitional-module">ml.rl.workflow.transitional module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="ml.rl.workflow.html#module-ml.rl.workflow">Module contents</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.caffe_utils">ml.rl.caffe_utils module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.debug_on_error">ml.rl.debug_on_error module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.json_serialize">ml.rl.json_serialize module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#ml-rl-parameters-module">ml.rl.parameters module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.tensorboardX">ml.rl.tensorboardX module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.torch_utils">ml.rl.torch_utils module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl.types">ml.rl.types module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.html#module-ml.rl">Module contents</a></li>
+</ul>
+</li>
+</ul>
+</div>
+</section>
+<section id="module-ml">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="modules.html" class="btn btn-neutral float-left" title="ml" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.html" class="btn btn-neutral float-right" title="ml.rl package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,314 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.evaluation package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.models package" href="ml.rl.models.html" />
+    <link rel="prev" title="Continuous Integration" href="../continuous_integration.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Evaluation</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.evaluation.cpe">ml.rl.evaluation.cpe module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-doubly-robust-estimator-module">ml.rl.evaluation.doubly_robust_estimator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-evaluation-data-page-module">ml.rl.evaluation.evaluation_data_page module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-evaluator-module">ml.rl.evaluation.evaluator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-ranking-evaluator-module">ml.rl.evaluation.ranking_evaluator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.evaluation.reward_net_evaluator">ml.rl.evaluation.reward_net_evaluator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-sequential-doubly-robust-estimator-module">ml.rl.evaluation.sequential_doubly_robust_estimator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-weighted-sequential-doubly-robust-estimator-module">ml.rl.evaluation.weighted_sequential_doubly_robust_estimator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-evaluation-world-model-evaluator-module">ml.rl.evaluation.world_model_evaluator module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.evaluation">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+      <li>ml.rl.evaluation package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.evaluation.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-evaluation-package">
+<h1>ml.rl.evaluation package<a class="headerlink" href="#ml-rl-evaluation-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.evaluation.cpe">
+<span id="ml-rl-evaluation-cpe-module"></span><h2>ml.rl.evaluation.cpe module<a class="headerlink" href="#module-ml.rl.evaluation.cpe" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeDetails">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.evaluation.cpe.</span></span><span class="sig-name descname"><span class="pre">CpeDetails</span></span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeDetails" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeDetails.log">
+<span class="sig-name descname"><span class="pre">log</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeDetails.log" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeDetails.log_to_tensorboard">
+<span class="sig-name descname"><span class="pre">log_to_tensorboard</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeDetails.log_to_tensorboard" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimate">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.evaluation.cpe.</span></span><span class="sig-name descname"><span class="pre">CpeEstimate</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">raw</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">normalized</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">raw_std_error</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">normalized_std_error</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimate" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">NamedTuple</span></code></p>
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimate.normalized">
+<span class="sig-name descname"><span class="pre">normalized</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">float</span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimate.normalized" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 1</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimate.normalized_std_error">
+<span class="sig-name descname"><span class="pre">normalized_std_error</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">float</span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimate.normalized_std_error" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 3</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimate.raw">
+<span class="sig-name descname"><span class="pre">raw</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">float</span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimate.raw" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 0</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimate.raw_std_error">
+<span class="sig-name descname"><span class="pre">raw_std_error</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">float</span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimate.raw_std_error" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 2</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.evaluation.cpe.</span></span><span class="sig-name descname"><span class="pre">CpeEstimateSet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">direct_method</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">inverse_propensity</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">doubly_robust</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">sequential_doubly_robust</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">weighted_doubly_robust</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">magic</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">NamedTuple</span></code></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.check_estimates_exist">
+<span class="sig-name descname"><span class="pre">check_estimates_exist</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.check_estimates_exist" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.direct_method">
+<span class="sig-name descname"><span class="pre">direct_method</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.direct_method" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 0</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.doubly_robust">
+<span class="sig-name descname"><span class="pre">doubly_robust</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.doubly_robust" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 2</p>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.fill_empty_with_zero">
+<span class="sig-name descname"><span class="pre">fill_empty_with_zero</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.fill_empty_with_zero" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.inverse_propensity">
+<span class="sig-name descname"><span class="pre">inverse_propensity</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.inverse_propensity" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 1</p>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.log">
+<span class="sig-name descname"><span class="pre">log</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.log" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.log_to_tensorboard">
+<span class="sig-name descname"><span class="pre">log_to_tensorboard</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">metric_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.log_to_tensorboard" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.magic">
+<span class="sig-name descname"><span class="pre">magic</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.magic" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 5</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.sequential_doubly_robust">
+<span class="sig-name descname"><span class="pre">sequential_doubly_robust</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.sequential_doubly_robust" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 3</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.CpeEstimateSet.weighted_doubly_robust">
+<span class="sig-name descname"><span class="pre">weighted_doubly_robust</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#ml.rl.evaluation.cpe.CpeEstimate" title="ml.rl.evaluation.cpe.CpeEstimate"><span class="pre">ml.rl.evaluation.cpe.CpeEstimate</span></a><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#ml.rl.evaluation.cpe.CpeEstimateSet.weighted_doubly_robust" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 4</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="ml.rl.evaluation.cpe.bootstrapped_std_error_of_mean">
+<span class="sig-prename descclassname"><span class="pre">ml.rl.evaluation.cpe.</span></span><span class="sig-name descname"><span class="pre">bootstrapped_std_error_of_mean</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">sample_percent</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0.25</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">num_samples</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1000</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.cpe.bootstrapped_std_error_of_mean" title="Permalink to this definition"></a></dt>
+<dd><p>Compute bootstrapped standard error of mean of input data.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>data</strong> – Input data (1D torch tensor or numpy array).</p></li>
+<li><p><strong>sample_percent</strong> – Size of sample to use to calculate bootstrap statistic.</p></li>
+<li><p><strong>num_samples</strong> – Number of times to sample.</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+</section>
+<section id="ml-rl-evaluation-doubly-robust-estimator-module">
+<h2>ml.rl.evaluation.doubly_robust_estimator module<a class="headerlink" href="#ml-rl-evaluation-doubly-robust-estimator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-evaluation-evaluation-data-page-module">
+<h2>ml.rl.evaluation.evaluation_data_page module<a class="headerlink" href="#ml-rl-evaluation-evaluation-data-page-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-evaluation-evaluator-module">
+<h2>ml.rl.evaluation.evaluator module<a class="headerlink" href="#ml-rl-evaluation-evaluator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-evaluation-ranking-evaluator-module">
+<h2>ml.rl.evaluation.ranking_evaluator module<a class="headerlink" href="#ml-rl-evaluation-ranking-evaluator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.evaluation.reward_net_evaluator">
+<span id="ml-rl-evaluation-reward-net-evaluator-module"></span><h2>ml.rl.evaluation.reward_net_evaluator module<a class="headerlink" href="#module-ml.rl.evaluation.reward_net_evaluator" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.evaluation.reward_net_evaluator.</span></span><span class="sig-name descname"><span class="pre">RewardNetEvaluator</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">trainer</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="ml.rl.training.html#ml.rl.training.reward_network_trainer.RewardNetTrainer" title="ml.rl.training.reward_network_trainer.RewardNetTrainer"><span class="pre">ml.rl.training.reward_network_trainer.RewardNetTrainer</span></a></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+<p>Evaluate reward networks</p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator.evaluate">
+<span class="sig-name descname"><span class="pre">evaluate</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">eval_tdp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="ml.rl.html#ml.rl.types.PreprocessedTrainingBatch" title="ml.rl.types.PreprocessedTrainingBatch"><span class="pre">ml.rl.types.PreprocessedTrainingBatch</span></a></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator.evaluate" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator.evaluate_post_training">
+<span class="sig-name descname"><span class="pre">evaluate_post_training</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.evaluation.reward_net_evaluator.RewardNetEvaluator.evaluate_post_training" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+</section>
+<section id="ml-rl-evaluation-sequential-doubly-robust-estimator-module">
+<h2>ml.rl.evaluation.sequential_doubly_robust_estimator module<a class="headerlink" href="#ml-rl-evaluation-sequential-doubly-robust-estimator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-evaluation-weighted-sequential-doubly-robust-estimator-module">
+<h2>ml.rl.evaluation.weighted_sequential_doubly_robust_estimator module<a class="headerlink" href="#ml-rl-evaluation-weighted-sequential-doubly-robust-estimator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-evaluation-world-model-evaluator-module">
+<h2>ml.rl.evaluation.world_model_evaluator module<a class="headerlink" href="#ml-rl-evaluation-world-model-evaluator-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.evaluation">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.evaluation" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="../continuous_integration.html" class="btn btn-neutral float-left" title="Continuous Integration" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.models.html" class="btn btn-neutral float-right" title="ml.rl.models package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,188 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.polyfill package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="License" href="../license.html" />
+    <link rel="prev" title="ml.rl package" href="ml.rl.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="modules.html">All Modules</a><ul class="current">
+<li class="toctree-l2 current"><a class="reference internal" href="ml.html">ml package</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="ml.html#subpackages">Subpackages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="ml.rl.html">ml.rl package</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.html#module-ml">Module contents</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="modules.html">ml</a> &raquo;</li>
+          <li><a href="ml.html">ml package</a> &raquo;</li>
+          <li><a href="ml.rl.html">ml.rl package</a> &raquo;</li>
+      <li>ml.rl.polyfill package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.polyfill.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-polyfill-package">
+<h1>ml.rl.polyfill package<a class="headerlink" href="#ml-rl-polyfill-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.polyfill.decorators">
+<span id="ml-rl-polyfill-decorators-module"></span><h2>ml.rl.polyfill.decorators module<a class="headerlink" href="#module-ml.rl.polyfill.decorators" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.polyfill.decorators.ClassPropertyDescriptor">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.polyfill.decorators.</span></span><span class="sig-name descname"><span class="pre">ClassPropertyDescriptor</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">fget</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.polyfill.decorators.ClassPropertyDescriptor" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="ml.rl.polyfill.decorators.classproperty">
+<span class="sig-prename descclassname"><span class="pre">ml.rl.polyfill.decorators.</span></span><span class="sig-name descname"><span class="pre">classproperty</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.polyfill.decorators.classproperty" title="Permalink to this definition"></a></dt>
+<dd><p>Allow for getter property usage on classmethods</p>
+<p>cf: <a class="reference external" href="https://stackoverflow.com/questions/5189699/how-to-make-a-class-property">https://stackoverflow.com/questions/5189699/how-to-make-a-class-property</a></p>
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.polyfill.exceptions">
+<span id="ml-rl-polyfill-exceptions-module"></span><h2>ml.rl.polyfill.exceptions module<a class="headerlink" href="#module-ml.rl.polyfill.exceptions" title="Permalink to this headline"></a></h2>
+<dl class="py exception">
+<dt class="sig sig-object py" id="ml.rl.polyfill.exceptions.NoRetriesException">
+<em class="property"><span class="pre">exception</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.polyfill.exceptions.</span></span><span class="sig-name descname"><span class="pre">NoRetriesException</span></span><a class="headerlink" href="#ml.rl.polyfill.exceptions.NoRetriesException" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">Exception</span></code></p>
+</dd></dl>
+
+<dl class="py exception">
+<dt class="sig sig-object py" id="ml.rl.polyfill.exceptions.NonRetryableException">
+<em class="property"><span class="pre">exception</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.polyfill.exceptions.</span></span><span class="sig-name descname"><span class="pre">NonRetryableException</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">child_exception</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">prefix</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">''</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.polyfill.exceptions.NonRetryableException" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="#ml.rl.polyfill.exceptions.NoRetriesException" title="ml.rl.polyfill.exceptions.NoRetriesException"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.polyfill.exceptions.NoRetriesException</span></code></a></p>
+</dd></dl>
+
+<dl class="py exception">
+<dt class="sig sig-object py" id="ml.rl.polyfill.exceptions.NonRetryableTypeError">
+<em class="property"><span class="pre">exception</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.polyfill.exceptions.</span></span><span class="sig-name descname"><span class="pre">NonRetryableTypeError</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">type_error</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.polyfill.exceptions.NonRetryableTypeError" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">TypeError</span></code>, <a class="reference internal" href="#ml.rl.polyfill.exceptions.NonRetryableException" title="ml.rl.polyfill.exceptions.NonRetryableException"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.polyfill.exceptions.NonRetryableException</span></code></a></p>
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.polyfill.types">
+<span id="ml-rl-polyfill-types-module"></span><h2>ml.rl.polyfill.types module<a class="headerlink" href="#module-ml.rl.polyfill.types" title="Permalink to this headline"></a></h2>
+<p>Polyfills fblearner types</p>
+</section>
+<section id="module-ml.rl.polyfill">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.polyfill" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.html" class="btn btn-neutral float-left" title="ml.rl package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="../license.html" class="btn btn-neutral float-right" title="License" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,146 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.prediction package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.preprocessing package" href="ml.rl.preprocessing.html" />
+    <link rel="prev" title="ml.rl.models package" href="ml.rl.models.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Prediction</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-prediction-dqn-torch-predictor-module">ml.rl.prediction.dqn_torch_predictor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-prediction-predictor-wrapper-module">ml.rl.prediction.predictor_wrapper module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.prediction">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+      <li>ml.rl.prediction package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.prediction.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-prediction-package">
+<h1>ml.rl.prediction package<a class="headerlink" href="#ml-rl-prediction-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-prediction-dqn-torch-predictor-module">
+<h2>ml.rl.prediction.dqn_torch_predictor module<a class="headerlink" href="#ml-rl-prediction-dqn-torch-predictor-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-prediction-predictor-wrapper-module">
+<h2>ml.rl.prediction.predictor_wrapper module<a class="headerlink" href="#ml-rl-prediction-predictor-wrapper-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.prediction">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.prediction" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.models.html" class="btn btn-neutral float-left" title="ml.rl.models package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.preprocessing.html" class="btn btn-neutral float-right" title="ml.rl.preprocessing package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,167 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.preprocessing package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.readers package" href="ml.rl.readers.html" />
+    <link rel="prev" title="ml.rl.prediction package" href="ml.rl.prediction.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Preprocessing</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-preprocessing-batch-preprocessor-module">ml.rl.preprocessing.batch_preprocessor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.preprocessing.identify_types">ml.rl.preprocessing.identify_types module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-preprocessing-normalization-module">ml.rl.preprocessing.normalization module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-preprocessing-postprocessor-module">ml.rl.preprocessing.postprocessor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-preprocessing-preprocessor-module">ml.rl.preprocessing.preprocessor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-preprocessing-sparse-to-dense-module">ml.rl.preprocessing.sparse_to_dense module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.preprocessing">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+      <li>ml.rl.preprocessing package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.preprocessing.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-preprocessing-package">
+<h1>ml.rl.preprocessing package<a class="headerlink" href="#ml-rl-preprocessing-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-preprocessing-batch-preprocessor-module">
+<h2>ml.rl.preprocessing.batch_preprocessor module<a class="headerlink" href="#ml-rl-preprocessing-batch-preprocessor-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.preprocessing.identify_types">
+<span id="ml-rl-preprocessing-identify-types-module"></span><h2>ml.rl.preprocessing.identify_types module<a class="headerlink" href="#module-ml.rl.preprocessing.identify_types" title="Permalink to this headline"></a></h2>
+<dl class="py function">
+<dt class="sig sig-object py" id="ml.rl.preprocessing.identify_types.identify_type">
+<span class="sig-prename descclassname"><span class="pre">ml.rl.preprocessing.identify_types.</span></span><span class="sig-name descname"><span class="pre">identify_type</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">values</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">enum_threshold</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">100</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.preprocessing.identify_types.identify_type" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+</section>
+<section id="ml-rl-preprocessing-normalization-module">
+<h2>ml.rl.preprocessing.normalization module<a class="headerlink" href="#ml-rl-preprocessing-normalization-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-preprocessing-postprocessor-module">
+<h2>ml.rl.preprocessing.postprocessor module<a class="headerlink" href="#ml-rl-preprocessing-postprocessor-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-preprocessing-preprocessor-module">
+<h2>ml.rl.preprocessing.preprocessor module<a class="headerlink" href="#ml-rl-preprocessing-preprocessor-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-preprocessing-sparse-to-dense-module">
+<h2>ml.rl.preprocessing.sparse_to_dense module<a class="headerlink" href="#ml-rl-preprocessing-sparse-to-dense-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.preprocessing">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.preprocessing" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.prediction.html" class="btn btn-neutral float-left" title="ml.rl.prediction package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.readers.html" class="btn btn-neutral float-right" title="ml.rl.readers package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,312 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.readers package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.simulators package" href="ml.rl.simulators.html" />
+    <link rel="prev" title="ml.rl.preprocessing package" href="ml.rl.preprocessing.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Readers</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.readers.base">ml.rl.readers.base module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.readers.data_streamer">ml.rl.readers.data_streamer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.readers.json_dataset_reader">ml.rl.readers.json_dataset_reader module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.readers.nparray_reader">ml.rl.readers.nparray_reader module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.readers">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+      <li>ml.rl.readers package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.readers.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-readers-package">
+<h1>ml.rl.readers package<a class="headerlink" href="#ml-rl-readers-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.readers.base">
+<span id="ml-rl-readers-base-module"></span><h2>ml.rl.readers.base module<a class="headerlink" href="#module-ml.rl.readers.base" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.base.ReaderBase">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.base.</span></span><span class="sig-name descname"><span class="pre">ReaderBase</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">batch_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">drop_small</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">num_shards</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.base.ReaderBase" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.base.ReaderBase.do_get_shard">
+<span class="sig-name descname"><span class="pre">do_get_shard</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">shard_id</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.base.ReaderBase.do_get_shard" title="Permalink to this definition"></a></dt>
+<dd><p>Subclass should implement this if the reader is shardable</p>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.base.ReaderBase.get_shard">
+<span class="sig-name descname"><span class="pre">get_shard</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">shard_id</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.base.ReaderBase.get_shard" title="Permalink to this definition"></a></dt>
+<dd><p>Returns a shard of this reader</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.base.ReaderIter">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.base.</span></span><span class="sig-name descname"><span class="pre">ReaderIter</span></span><a class="headerlink" href="#ml.rl.readers.base.ReaderIter" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.base.ReaderIter.read_batch">
+<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">read_batch</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">collections.OrderedDict</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#ml.rl.readers.base.ReaderIter.read_batch" title="Permalink to this definition"></a></dt>
+<dd><p>Read a batch of data. The return value should be an OrderedDict.
+Returns None when there is no more data.</p>
+</dd></dl>
+
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.readers.data_streamer">
+<span id="ml-rl-readers-data-streamer-module"></span><h2>ml.rl.readers.data_streamer module<a class="headerlink" href="#module-ml.rl.readers.data_streamer" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.data_streamer.DataStreamer">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.data_streamer.</span></span><span class="sig-name descname"><span class="pre">DataStreamer</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data_reader</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">num_workers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">pin_memory</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">worker_init_fn</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.data_streamer.DataStreamer" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
+<p>Data streamer. Provides single- or multi-process iterators over the data_reader.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>data_reader</strong> (<em>DataReader</em>) – data_reader from which to stream the data.</p></li>
+<li><p><strong>num_workers</strong> (<em>int</em><em>, </em><em>optional</em>) – how many subprocesses to use for data
+loading. 0 means that the data will be loaded in the main process.
+(default: 0)</p></li>
+<li><p><strong>pin_memory</strong> (<em>bool</em><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the data streamer will copy tensors
+into CUDA pinned memory before returning them.</p></li>
+<li><p><strong>timeout</strong> (<em>numeric</em><em>, </em><em>optional</em>) – if positive, the timeout value for collecting a
+batch from workers. Should always be non-negative. (default: 0)</p></li>
+<li><p><strong>worker_init_fn</strong> (<em>callable</em><em>, </em><em>optional</em>) – If not None, this will be called on each
+worker subprocess with the worker id (an int in <code class="docutils literal notranslate"><span class="pre">[0,</span> <span class="pre">num_workers</span> <span class="pre">-</span> <span class="pre">1]</span></code>) as
+input, after seeding and before data loading. (default: None)</p></li>
+</ul>
+</dd>
+</dl>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>By default, each worker will have its PyTorch seed set to
+<code class="docutils literal notranslate"><span class="pre">base_seed</span> <span class="pre">+</span> <span class="pre">worker_id</span></code>, where <code class="docutils literal notranslate"><span class="pre">base_seed</span></code> is a long generated
+by main process using its RNG. However, seeds for other libraries
+may be duplicated upon initializing workers (w.g., NumPy), causing
+each worker to return identical random numbers. (See
+<span class="xref std std-ref">datastreamer-workers-random-seed</span> section in FAQ.) You may
+use <code class="docutils literal notranslate"><span class="pre">torch.initial_seed()</span></code> to access the PyTorch seed for each
+worker in <code class="xref py py-attr docutils literal notranslate"><span class="pre">worker_init_fn</span></code>, and use it to set other seeds
+before data loading.</p>
+</div>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>If <code class="docutils literal notranslate"><span class="pre">spawn</span></code> start method is used, <code class="xref py py-attr docutils literal notranslate"><span class="pre">worker_init_fn</span></code> cannot be an
+unpickleable object, e.g., a lambda function.</p>
+</div>
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.data_streamer.WorkerDone">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.data_streamer.</span></span><span class="sig-name descname"><span class="pre">WorkerDone</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">worker_id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.data_streamer.WorkerDone" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">tuple</span></code></p>
+<dl class="py attribute">
+<dt class="sig sig-object py" id="ml.rl.readers.data_streamer.WorkerDone.worker_id">
+<span class="sig-name descname"><span class="pre">worker_id</span></span><a class="headerlink" href="#ml.rl.readers.data_streamer.WorkerDone.worker_id" title="Permalink to this definition"></a></dt>
+<dd><p>Alias for field number 0</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="ml.rl.readers.data_streamer.pin_memory">
+<span class="sig-prename descclassname"><span class="pre">ml.rl.readers.data_streamer.</span></span><span class="sig-name descname"><span class="pre">pin_memory</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">batch</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.data_streamer.pin_memory" title="Permalink to this definition"></a></dt>
+<dd><p>This is ripped off from dataloader. The only difference is that it preserves
+the type of Mapping so that the OrderedDict is maintained.</p>
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.readers.json_dataset_reader">
+<span id="ml-rl-readers-json-dataset-reader-module"></span><h2>ml.rl.readers.json_dataset_reader module<a class="headerlink" href="#module-ml.rl.readers.json_dataset_reader" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReader">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.json_dataset_reader.</span></span><span class="sig-name descname"><span class="pre">JSONDatasetReader</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">batch_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">preprocess_handler</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReader" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="#ml.rl.readers.base.ReaderBase" title="ml.rl.readers.base.ReaderBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.readers.base.ReaderBase</span></code></a></p>
+<p>Create the reader for a JSON training dataset.</p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReader.line_count">
+<span class="sig-name descname"><span class="pre">line_count</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReader.line_count" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReader.read_all">
+<span class="sig-name descname"><span class="pre">read_all</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReader.read_all" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReader.read_batch">
+<span class="sig-name descname"><span class="pre">read_batch</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReader.read_batch" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReader.reset_iterator">
+<span class="sig-name descname"><span class="pre">reset_iterator</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReader.reset_iterator" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReaderIter">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.json_dataset_reader.</span></span><span class="sig-name descname"><span class="pre">JSONDatasetReaderIter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reader</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReaderIter" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="#ml.rl.readers.base.ReaderIter" title="ml.rl.readers.base.ReaderIter"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.readers.base.ReaderIter</span></code></a></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.json_dataset_reader.JSONDatasetReaderIter.read_batch">
+<span class="sig-name descname"><span class="pre">read_batch</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">collections.OrderedDict</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#ml.rl.readers.json_dataset_reader.JSONDatasetReaderIter.read_batch" title="Permalink to this definition"></a></dt>
+<dd><p>Read a batch of data. The return value should be an OrderedDict.
+Returns None when there is no more data.</p>
+</dd></dl>
+
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.readers.nparray_reader">
+<span id="ml-rl-readers-nparray-reader-module"></span><h2>ml.rl.readers.nparray_reader module<a class="headerlink" href="#module-ml.rl.readers.nparray_reader" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.nparray_reader.NpArrayReader">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.nparray_reader.</span></span><span class="sig-name descname"><span class="pre">NpArrayReader</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.nparray_reader.NpArrayReader" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="#ml.rl.readers.base.ReaderBase" title="ml.rl.readers.base.ReaderBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.readers.base.ReaderBase</span></code></a></p>
+<p>Basic reader taking <cite>np.ndarray`s of a whole dataset and split them into
+chunks of `batch_size</cite>.</p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.nparray_reader.NpArrayReader.do_get_shard">
+<span class="sig-name descname"><span class="pre">do_get_shard</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">shard_id</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.nparray_reader.NpArrayReader.do_get_shard" title="Permalink to this definition"></a></dt>
+<dd><p>Subclass should implement this if the reader is shardable</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt class="sig sig-object py" id="ml.rl.readers.nparray_reader.NpArrayReaderIter">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">ml.rl.readers.nparray_reader.</span></span><span class="sig-name descname"><span class="pre">NpArrayReaderIter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reader</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#ml.rl.readers.nparray_reader.NpArrayReaderIter" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="#ml.rl.readers.base.ReaderIter" title="ml.rl.readers.base.ReaderIter"><code class="xref py py-class docutils literal notranslate"><span class="pre">ml.rl.readers.base.ReaderIter</span></code></a></p>
+<dl class="py method">
+<dt class="sig sig-object py" id="ml.rl.readers.nparray_reader.NpArrayReaderIter.read_batch">
+<span class="sig-name descname"><span class="pre">read_batch</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">collections.OrderedDict</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#ml.rl.readers.nparray_reader.NpArrayReaderIter.read_batch" title="Permalink to this definition"></a></dt>
+<dd><p>Read a batch of data. The return value should be an OrderedDict.
+Returns None when there is no more data.</p>
+</dd></dl>
+
+</dd></dl>
+
+</section>
+<section id="module-ml.rl.readers">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.readers" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.preprocessing.html" class="btn btn-neutral float-left" title="ml.rl.preprocessing package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.simulators.html" class="btn btn-neutral float-right" title="ml.rl.simulators package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,174 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.training.gradient_free package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.training.ranking package" href="ml.rl.training.ranking.html" />
+    <link rel="prev" title="ml.rl.training package" href="ml.rl.training.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="ml.rl.training.html">Training</a><ul class="current">
+<li class="toctree-l2 current"><a class="reference internal" href="ml.rl.training.html#subpackages">Subpackages</a><ul class="current">
+<li class="toctree-l3 current"><a class="current reference internal" href="#">ml.rl.training.gradient_free package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#ml-rl-training-gradient-free-es-worker-module">ml.rl.training.gradient_free.es_worker module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#ml-rl-training-gradient-free-evolution-pool-module">ml.rl.training.gradient_free.evolution_pool module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#module-ml.rl.training.gradient_free">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.ranking.html">ml.rl.training.ranking package</a></li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.world_model.html">ml.rl.training.world_model package</a></li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-c51-trainer-module">ml.rl.training.c51_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-cem-trainer-module">ml.rl.training.cem_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-module">ml.rl.training.dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-base-module">ml.rl.training.dqn_trainer_base module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-imitator-training-module">ml.rl.training.imitator_training module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.loss_reporter">ml.rl.training.loss_reporter module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.on_policy_predictor">ml.rl.training.on_policy_predictor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-parametric-dqn-trainer-module">ml.rl.training.parametric_dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-qrdqn-trainer-module">ml.rl.training.qrdqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.reward_network_trainer">ml.rl.training.reward_network_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.rl_dataset">ml.rl.training.rl_dataset module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-rl-trainer-pytorch-module">ml.rl.training.rl_trainer_pytorch module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-sac-trainer-module">ml.rl.training.sac_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-slate-q-trainer-module">ml.rl.training.slate_q_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-td3-trainer-module">ml.rl.training.td3_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.trainer">ml.rl.training.trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.training_data_page">ml.rl.training.training_data_page module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="ml.rl.training.html">ml.rl.training package</a> &raquo;</li>
+      <li>ml.rl.training.gradient_free package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.training.gradient_free.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-training-gradient-free-package">
+<h1>ml.rl.training.gradient_free package<a class="headerlink" href="#ml-rl-training-gradient-free-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-training-gradient-free-es-worker-module">
+<h2>ml.rl.training.gradient_free.es_worker module<a class="headerlink" href="#ml-rl-training-gradient-free-es-worker-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-training-gradient-free-evolution-pool-module">
+<h2>ml.rl.training.gradient_free.evolution_pool module<a class="headerlink" href="#ml-rl-training-gradient-free-evolution-pool-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.training.gradient_free">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.training.gradient_free" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.training.html" class="btn btn-neutral float-left" title="ml.rl.training package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.training.ranking.html" class="btn btn-neutral float-right" title="ml.rl.training.ranking package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,174 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.training.ranking package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.training.world_model package" href="ml.rl.training.world_model.html" />
+    <link rel="prev" title="ml.rl.training.gradient_free package" href="ml.rl.training.gradient_free.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="ml.rl.training.html">Training</a><ul class="current">
+<li class="toctree-l2 current"><a class="reference internal" href="ml.rl.training.html#subpackages">Subpackages</a><ul class="current">
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.gradient_free.html">ml.rl.training.gradient_free package</a></li>
+<li class="toctree-l3 current"><a class="current reference internal" href="#">ml.rl.training.ranking package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#ml-rl-training-ranking-seq2slate-tf-trainer-module">ml.rl.training.ranking.seq2slate_tf_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#ml-rl-training-ranking-seq2slate-trainer-module">ml.rl.training.ranking.seq2slate_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#module-ml.rl.training.ranking">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.world_model.html">ml.rl.training.world_model package</a></li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-c51-trainer-module">ml.rl.training.c51_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-cem-trainer-module">ml.rl.training.cem_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-module">ml.rl.training.dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-base-module">ml.rl.training.dqn_trainer_base module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-imitator-training-module">ml.rl.training.imitator_training module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.loss_reporter">ml.rl.training.loss_reporter module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.on_policy_predictor">ml.rl.training.on_policy_predictor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-parametric-dqn-trainer-module">ml.rl.training.parametric_dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-qrdqn-trainer-module">ml.rl.training.qrdqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.reward_network_trainer">ml.rl.training.reward_network_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.rl_dataset">ml.rl.training.rl_dataset module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-rl-trainer-pytorch-module">ml.rl.training.rl_trainer_pytorch module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-sac-trainer-module">ml.rl.training.sac_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-slate-q-trainer-module">ml.rl.training.slate_q_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-td3-trainer-module">ml.rl.training.td3_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.trainer">ml.rl.training.trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.training_data_page">ml.rl.training.training_data_page module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="ml.rl.training.html">ml.rl.training package</a> &raquo;</li>
+      <li>ml.rl.training.ranking package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.training.ranking.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-training-ranking-package">
+<h1>ml.rl.training.ranking package<a class="headerlink" href="#ml-rl-training-ranking-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-training-ranking-seq2slate-tf-trainer-module">
+<h2>ml.rl.training.ranking.seq2slate_tf_trainer module<a class="headerlink" href="#ml-rl-training-ranking-seq2slate-tf-trainer-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-training-ranking-seq2slate-trainer-module">
+<h2>ml.rl.training.ranking.seq2slate_trainer module<a class="headerlink" href="#ml-rl-training-ranking-seq2slate-trainer-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.training.ranking">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.training.ranking" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.training.gradient_free.html" class="btn btn-neutral float-left" title="ml.rl.training.gradient_free package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.training.world_model.html" class="btn btn-neutral float-right" title="ml.rl.training.world_model package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.training.world_model package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml.rl.workflow package" href="ml.rl.workflow.html" />
+    <link rel="prev" title="ml.rl.training.ranking package" href="ml.rl.training.ranking.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="ml.rl.training.html">Training</a><ul class="current">
+<li class="toctree-l2 current"><a class="reference internal" href="ml.rl.training.html#subpackages">Subpackages</a><ul class="current">
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.gradient_free.html">ml.rl.training.gradient_free package</a></li>
+<li class="toctree-l3"><a class="reference internal" href="ml.rl.training.ranking.html">ml.rl.training.ranking package</a></li>
+<li class="toctree-l3 current"><a class="current reference internal" href="#">ml.rl.training.world_model package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#ml-rl-training-world-model-mdnrnn-trainer-module">ml.rl.training.world_model.mdnrnn_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#module-ml.rl.training.world_model">Module contents</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-c51-trainer-module">ml.rl.training.c51_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-cem-trainer-module">ml.rl.training.cem_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-module">ml.rl.training.dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-dqn-trainer-base-module">ml.rl.training.dqn_trainer_base module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-imitator-training-module">ml.rl.training.imitator_training module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.loss_reporter">ml.rl.training.loss_reporter module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.on_policy_predictor">ml.rl.training.on_policy_predictor module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-parametric-dqn-trainer-module">ml.rl.training.parametric_dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-qrdqn-trainer-module">ml.rl.training.qrdqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.reward_network_trainer">ml.rl.training.reward_network_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.rl_dataset">ml.rl.training.rl_dataset module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-rl-trainer-pytorch-module">ml.rl.training.rl_trainer_pytorch module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-sac-trainer-module">ml.rl.training.sac_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-slate-q-trainer-module">ml.rl.training.slate_q_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#ml-rl-training-td3-trainer-module">ml.rl.training.td3_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.trainer">ml.rl.training.trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training.training_data_page">ml.rl.training.training_data_page module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="ml.rl.training.html#module-ml.rl.training">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="ml.rl.training.html">ml.rl.training package</a> &raquo;</li>
+      <li>ml.rl.training.world_model package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.training.world_model.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-training-world-model-package">
+<h1>ml.rl.training.world_model package<a class="headerlink" href="#ml-rl-training-world-model-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-training-world-model-mdnrnn-trainer-module">
+<h2>ml.rl.training.world_model.mdnrnn_trainer module<a class="headerlink" href="#ml-rl-training-world-model-mdnrnn-trainer-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.training.world_model">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.training.world_model" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.training.ranking.html" class="btn btn-neutral float-left" title="ml.rl.training.ranking package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="ml.rl.workflow.html" class="btn btn-neutral float-right" title="ml.rl.workflow package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>ml.rl.workflow package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="ml" href="modules.html" />
+    <link rel="prev" title="ml.rl.training.world_model package" href="ml.rl.training.world_model.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../distributed.html">Distributed Training</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.readers.html">Readers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.simulators.html">Simulators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ml.rl.training.html">Training</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">Workflow</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-base-workflow-module">ml.rl.workflow.base_workflow module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-create-normalization-metadata-module">ml.rl.workflow.create_normalization_metadata module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-dqn-workflow-module">ml.rl.workflow.dqn_workflow module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-helpers-module">ml.rl.workflow.helpers module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-page-handler-module">ml.rl.workflow.page_handler module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-parametric-dqn-workflow-module">ml.rl.workflow.parametric_dqn_workflow module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-preprocess-handler-module">ml.rl.workflow.preprocess_handler module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#ml-rl-workflow-transitional-module">ml.rl.workflow.transitional module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#module-ml.rl.workflow">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+      <li>ml.rl.workflow package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/ml.rl.workflow.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="ml-rl-workflow-package">
+<h1>ml.rl.workflow package<a class="headerlink" href="#ml-rl-workflow-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-base-workflow-module">
+<h2>ml.rl.workflow.base_workflow module<a class="headerlink" href="#ml-rl-workflow-base-workflow-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-create-normalization-metadata-module">
+<h2>ml.rl.workflow.create_normalization_metadata module<a class="headerlink" href="#ml-rl-workflow-create-normalization-metadata-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-dqn-workflow-module">
+<h2>ml.rl.workflow.dqn_workflow module<a class="headerlink" href="#ml-rl-workflow-dqn-workflow-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-helpers-module">
+<h2>ml.rl.workflow.helpers module<a class="headerlink" href="#ml-rl-workflow-helpers-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-page-handler-module">
+<h2>ml.rl.workflow.page_handler module<a class="headerlink" href="#ml-rl-workflow-page-handler-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-parametric-dqn-workflow-module">
+<h2>ml.rl.workflow.parametric_dqn_workflow module<a class="headerlink" href="#ml-rl-workflow-parametric-dqn-workflow-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-preprocess-handler-module">
+<h2>ml.rl.workflow.preprocess_handler module<a class="headerlink" href="#ml-rl-workflow-preprocess-handler-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="ml-rl-workflow-transitional-module">
+<h2>ml.rl.workflow.transitional module<a class="headerlink" href="#ml-rl-workflow-transitional-module" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-ml.rl.workflow">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-ml.rl.workflow" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="ml.rl.training.world_model.html" class="btn btn-neutral float-left" title="ml.rl.training.world_model package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="modules.html" class="btn btn-neutral float-right" title="ml" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2019, Facebook Inc..</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
@@ -0,0 +1,461 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>reagent.training.cb package &mdash; ReAgent 1.0 documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
+        <script src="../_static/jquery.js"></script>
+        <script src="../_static/underscore.js"></script>
+        <script src="../_static/doctools.js"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="reagent.training.cfeval package" href="reagent.training.cfeval.html" />
+    <link rel="prev" title="reagent.training package" href="reagent.training.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+            <a href="../index.html" class="icon icon-home"> ReAgent
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Usage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../rasp_tutorial.html">RASP (Not Actively Maintained)</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Advanced Topics</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../continuous_integration.html">Continuous Integration</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Package Reference</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="reagent.core.html">Core</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.data.html">Data</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.gym.html">Gym</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.evaluation.html">Evaluation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.lite.html">Lite</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.mab.html">MAB</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.model_managers.html">Model Managers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.model_utils.html">Model Utils</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.net_builder.html">Net Builders</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.optimizer.html">Optimizers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.models.html">Models</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.prediction.html">Prediction</a></li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.preprocessing.html">Preprocessing</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="reagent.training.html">Training</a><ul class="current">
+<li class="toctree-l2 current"><a class="reference internal" href="reagent.training.html#subpackages">Subpackages</a><ul class="current">
+<li class="toctree-l3 current"><a class="current reference internal" href="#">reagent.training.cb package</a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#submodules">Submodules</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#module-reagent.training.cb.linucb_trainer">reagent.training.cb.linucb_trainer module</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#module-reagent.training.cb">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="reagent.training.cfeval.html">reagent.training.cfeval package</a></li>
+<li class="toctree-l3"><a class="reference internal" href="reagent.training.gradient_free.html">reagent.training.gradient_free package</a></li>
+<li class="toctree-l3"><a class="reference internal" href="reagent.training.ranking.html">reagent.training.ranking package</a></li>
+<li class="toctree-l3"><a class="reference internal" href="reagent.training.world_model.html">reagent.training.world_model package</a></li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#submodules">Submodules</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.c51_trainer">reagent.training.c51_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.cem_trainer">reagent.training.cem_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.discrete_crr_trainer">reagent.training.discrete_crr_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.dqn_trainer">reagent.training.dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.dqn_trainer_base">reagent.training.dqn_trainer_base module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.imitator_training">reagent.training.imitator_training module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.multi_stage_trainer">reagent.training.multi_stage_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.parameters">reagent.training.parameters module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.parametric_dqn_trainer">reagent.training.parametric_dqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.ppo_trainer">reagent.training.ppo_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.qrdqn_trainer">reagent.training.qrdqn_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.reagent_lightning_module">reagent.training.reagent_lightning_module module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.reinforce_trainer">reagent.training.reinforce_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.reward_network_trainer">reagent.training.reward_network_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.rl_trainer_pytorch">reagent.training.rl_trainer_pytorch module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.sac_trainer">reagent.training.sac_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.slate_q_trainer">reagent.training.slate_q_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.td3_trainer">reagent.training.td3_trainer module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training.utils">reagent.training.utils module</a></li>
+<li class="toctree-l2"><a class="reference internal" href="reagent.training.html#module-reagent.training">Module contents</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="reagent.workflow.html">Workflow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="modules.html">All Modules</a></li>
+</ul>
+<p class="caption" role="heading"><span class="caption-text">Others</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/facebookresearch/ReAgent">Github</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">ReAgent</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
+          <li><a href="reagent.training.html">reagent.training package</a> &raquo;</li>
+      <li>reagent.training.cb package</li>
+      <li class="wy-breadcrumbs-aside">
+            <a href="../_sources/api/reagent.training.cb.rst.txt" rel="nofollow"> View page source</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="reagent-training-cb-package">
+<h1>reagent.training.cb package<a class="headerlink" href="#reagent-training-cb-package" title="Permalink to this headline"></a></h1>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline"></a></h2>
+</section>
+<section id="module-reagent.training.cb.linucb_trainer">
+<span id="reagent-training-cb-linucb-trainer-module"></span><h2>reagent.training.cb.linucb_trainer module<a class="headerlink" href="#module-reagent.training.cb.linucb_trainer" title="Permalink to this headline"></a></h2>
+<dl class="py class">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">reagent.training.cb.linucb_trainer.</span></span><span class="sig-name descname"><span class="pre">LinUCBTrainer</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">policy</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="reagent.gym.policies.html#reagent.gym.policies.policy.Policy" title="reagent.gym.policies.policy.Policy"><span class="pre">reagent.gym.policies.policy.Policy</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">num_actions</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">-</span> <span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_interaction_features</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference internal" href="reagent.training.html#reagent.training.reagent_lightning_module.ReAgentLightningModule" title="reagent.training.reagent_lightning_module.ReAgentLightningModule"><code class="xref py py-class docutils literal notranslate"><span class="pre">reagent.training.reagent_lightning_module.ReAgentLightningModule</span></code></a></p>
+<p>The trainer for LinUCB Contextual Bandit model.
+The model estimates a ridge regression (linear) and only supports dense features.
+The actions are assumed to be one of:</p>
+<blockquote>
+<div><ul class="simple">
+<li><dl class="simple">
+<dt>Fixed actions. The same (have the same semantic meaning) actions across all contexts.</dt><dd><p>If actions are fixed, they can’t have features associated with them.</p>
+</dd>
+</dl>
+</li>
+<li><dl class="simple">
+<dt>Feature actions. We can have different number and identities of actions in each</dt><dd><p>context. The actions must have features to represent their semantic meaning.</p>
+</dd>
+</dl>
+</li>
+</ul>
+</div></blockquote>
+<p>Reference: <a class="reference external" href="https://arxiv.org/pdf/1003.0146.pdf">https://arxiv.org/pdf/1003.0146.pdf</a></p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>policy</strong> – The policy to be trained. Its scorer has to be LinearRegressionUCB</p></li>
+<li><p><strong>num_actions</strong> – The number of actions. If num_actions==-1, the actions are assumed to be feature actions,
+otherwise they are assumed to be fixed actions.</p></li>
+<li><p><strong>use_interaction_features</strong> – If True,</p></li>
+</ul>
+</dd>
+</dl>
+<dl class="py attribute">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.allow_zero_length_dataloader_with_multiple_devices">
+<span class="sig-name descname"><span class="pre">allow_zero_length_dataloader_with_multiple_devices</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.allow_zero_length_dataloader_with_multiple_devices" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.configure_optimizers">
+<span class="sig-name descname"><span class="pre">configure_optimizers</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.configure_optimizers" title="Permalink to this definition"></a></dt>
+<dd><p>Choose what optimizers and learning-rate schedulers to use in your optimization.
+Normally you’d need one. But in the case of GANs or similar you might have multiple.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>Any of these 6 options.</p>
+<ul class="simple">
+<li><p><strong>Single optimizer</strong>.</p></li>
+<li><p><strong>List or Tuple</strong> of optimizers.</p></li>
+<li><p><strong>Two lists</strong> - The first list has multiple optimizers, and the second has multiple LR schedulers
+(or multiple <code class="docutils literal notranslate"><span class="pre">lr_scheduler_config</span></code>).</p></li>
+<li><p><strong>Dictionary</strong>, with an <code class="docutils literal notranslate"><span class="pre">&quot;optimizer&quot;</span></code> key, and (optionally) a <code class="docutils literal notranslate"><span class="pre">&quot;lr_scheduler&quot;</span></code>
+key whose value is a single LR scheduler or <code class="docutils literal notranslate"><span class="pre">lr_scheduler_config</span></code>.</p></li>
+<li><p><strong>Tuple of dictionaries</strong> as described above, with an optional <code class="docutils literal notranslate"><span class="pre">&quot;frequency&quot;</span></code> key.</p></li>
+<li><p><strong>None</strong> - Fit will run without any optimizer.</p></li>
+</ul>
+</p>
+</dd>
+</dl>
+<p>The <code class="docutils literal notranslate"><span class="pre">lr_scheduler_config</span></code> is a dictionary which contains the scheduler and its associated configuration.
+The default configuration is shown below.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">lr_scheduler_config</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="c1"># REQUIRED: The scheduler instance</span>
+    <span class="s2">&quot;scheduler&quot;</span><span class="p">:</span> <span class="n">lr_scheduler</span><span class="p">,</span>
+    <span class="c1"># The unit of the scheduler&#39;s step size, could also be &#39;step&#39;.</span>
+    <span class="c1"># &#39;epoch&#39; updates the scheduler on epoch end whereas &#39;step&#39;</span>
+    <span class="c1"># updates it after a optimizer update.</span>
+    <span class="s2">&quot;interval&quot;</span><span class="p">:</span> <span class="s2">&quot;epoch&quot;</span><span class="p">,</span>
+    <span class="c1"># How many epochs/steps should pass between calls to</span>
+    <span class="c1"># `scheduler.step()`. 1 corresponds to updating the learning</span>
+    <span class="c1"># rate after every epoch/step.</span>
+    <span class="s2">&quot;frequency&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="c1"># Metric to to monitor for schedulers like `ReduceLROnPlateau`</span>
+    <span class="s2">&quot;monitor&quot;</span><span class="p">:</span> <span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
+    <span class="c1"># If set to `True`, will enforce that the value specified &#39;monitor&#39;</span>
+    <span class="c1"># is available when the scheduler is updated, thus stopping</span>
+    <span class="c1"># training if not found. If set to `False`, it will only produce a warning</span>
+    <span class="s2">&quot;strict&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="c1"># If using the `LearningRateMonitor` callback to monitor the</span>
+    <span class="c1"># learning rate progress, this keyword can be used to specify</span>
+    <span class="c1"># a custom logged name</span>
+    <span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>When there are schedulers in which the <code class="docutils literal notranslate"><span class="pre">.step()</span></code> method is conditioned on a value, such as the
+<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.optim.lr_scheduler.ReduceLROnPlateau</span></code> scheduler, Lightning requires that the
+<code class="docutils literal notranslate"><span class="pre">lr_scheduler_config</span></code> contains the keyword <code class="docutils literal notranslate"><span class="pre">&quot;monitor&quot;</span></code> set to the metric name that the scheduler
+should be conditioned on.</p>
+<p>Metrics can be made available to monitor by simply logging it using
+<code class="docutils literal notranslate"><span class="pre">self.log('metric_to_track',</span> <span class="pre">metric_val)</span></code> in your <code class="xref py py-class docutils literal notranslate"><span class="pre">LightningModule</span></code>.</p>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>The <code class="docutils literal notranslate"><span class="pre">frequency</span></code> value specified in a dict along with the <code class="docutils literal notranslate"><span class="pre">optimizer</span></code> key is an int corresponding
+to the number of sequential batches optimized with the specific optimizer.
+It should be given to none or to all of the optimizers.
+There is a difference between passing multiple optimizers in a list,
+and passing multiple optimizers in dictionaries with a frequency of 1:</p>
+<blockquote>
+<div><ul class="simple">
+<li><p>In the former case, all optimizers will operate on the given batch in each optimization step.</p></li>
+<li><p>In the latter, only one optimizer will operate on the given batch at every step.</p></li>
+</ul>
+</div></blockquote>
+<p>This is different from the <code class="docutils literal notranslate"><span class="pre">frequency</span></code> value specified in the <code class="docutils literal notranslate"><span class="pre">lr_scheduler_config</span></code> mentioned above.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">optimizer_one</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="n">optimizer_two</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="k">return</span> <span class="p">[</span>
+        <span class="p">{</span><span class="s2">&quot;optimizer&quot;</span><span class="p">:</span> <span class="n">optimizer_one</span><span class="p">,</span> <span class="s2">&quot;frequency&quot;</span><span class="p">:</span> <span class="mi">5</span><span class="p">},</span>
+        <span class="p">{</span><span class="s2">&quot;optimizer&quot;</span><span class="p">:</span> <span class="n">optimizer_two</span><span class="p">,</span> <span class="s2">&quot;frequency&quot;</span><span class="p">:</span> <span class="mi">10</span><span class="p">},</span>
+    <span class="p">]</span>
+</pre></div>
+</div>
+<p>In this example, the first optimizer will be used for the first 5 steps,
+the second optimizer for the next 10 steps and that cycle will continue.
+If an LR scheduler is specified for an optimizer using the <code class="docutils literal notranslate"><span class="pre">lr_scheduler</span></code> key in the above dict,
+the scheduler will only be updated when its optimizer is being used.</p>
+</div>
+<p>Examples:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># most cases. no learning rate scheduler</span>
+<span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">)</span>
+
+<span class="c1"># multiple optimizer case (e.g.: GAN)</span>
+<span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">gen_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_gen</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="n">dis_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_dis</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.02</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">gen_opt</span><span class="p">,</span> <span class="n">dis_opt</span>
+
+<span class="c1"># example with learning rate schedulers</span>
+<span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">gen_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_gen</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="n">dis_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_dis</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.02</span><span class="p">)</span>
+    <span class="n">dis_sch</span> <span class="o">=</span> <span class="n">CosineAnnealing</span><span class="p">(</span><span class="n">dis_opt</span><span class="p">,</span> <span class="n">T_max</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
+    <span class="k">return</span> <span class="p">[</span><span class="n">gen_opt</span><span class="p">,</span> <span class="n">dis_opt</span><span class="p">],</span> <span class="p">[</span><span class="n">dis_sch</span><span class="p">]</span>
+
+<span class="c1"># example with step-based learning rate schedulers</span>
+<span class="c1"># each optimizer has its own scheduler</span>
+<span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">gen_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_gen</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="n">dis_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_dis</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.02</span><span class="p">)</span>
+    <span class="n">gen_sch</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s1">&#39;scheduler&#39;</span><span class="p">:</span> <span class="n">ExponentialLR</span><span class="p">(</span><span class="n">gen_opt</span><span class="p">,</span> <span class="mf">0.99</span><span class="p">),</span>
+        <span class="s1">&#39;interval&#39;</span><span class="p">:</span> <span class="s1">&#39;step&#39;</span>  <span class="c1"># called after each training step</span>
+    <span class="p">}</span>
+    <span class="n">dis_sch</span> <span class="o">=</span> <span class="n">CosineAnnealing</span><span class="p">(</span><span class="n">dis_opt</span><span class="p">,</span> <span class="n">T_max</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span> <span class="c1"># called every epoch</span>
+    <span class="k">return</span> <span class="p">[</span><span class="n">gen_opt</span><span class="p">,</span> <span class="n">dis_opt</span><span class="p">],</span> <span class="p">[</span><span class="n">gen_sch</span><span class="p">,</span> <span class="n">dis_sch</span><span class="p">]</span>
+
+<span class="c1"># example with optimizer frequencies</span>
+<span class="c1"># see training procedure in `Improved Training of Wasserstein GANs`, Algorithm 1</span>
+<span class="c1"># https://arxiv.org/abs/1704.00028</span>
+<span class="k">def</span> <span class="nf">configure_optimizers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">gen_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_gen</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
+    <span class="n">dis_opt</span> <span class="o">=</span> <span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_dis</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.02</span><span class="p">)</span>
+    <span class="n">n_critic</span> <span class="o">=</span> <span class="mi">5</span>
+    <span class="k">return</span> <span class="p">(</span>
+        <span class="p">{</span><span class="s1">&#39;optimizer&#39;</span><span class="p">:</span> <span class="n">dis_opt</span><span class="p">,</span> <span class="s1">&#39;frequency&#39;</span><span class="p">:</span> <span class="n">n_critic</span><span class="p">},</span>
+        <span class="p">{</span><span class="s1">&#39;optimizer&#39;</span><span class="p">:</span> <span class="n">gen_opt</span><span class="p">,</span> <span class="s1">&#39;frequency&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span>
+    <span class="p">)</span>
+</pre></div>
+</div>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>Some things to know:</p>
+<ul class="simple">
+<li><p>Lightning calls <code class="docutils literal notranslate"><span class="pre">.backward()</span></code> and <code class="docutils literal notranslate"><span class="pre">.step()</span></code> on each optimizer and learning rate scheduler as needed.</p></li>
+<li><p>If you use 16-bit precision (<code class="docutils literal notranslate"><span class="pre">precision=16</span></code>), Lightning will automatically handle the optimizers.</p></li>
+<li><p>If you use multiple optimizers, <a class="reference internal" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.training_step" title="reagent.training.cb.linucb_trainer.LinUCBTrainer.training_step"><code class="xref py py-meth docutils literal notranslate"><span class="pre">training_step()</span></code></a> will have an additional <code class="docutils literal notranslate"><span class="pre">optimizer_idx</span></code> parameter.</p></li>
+<li><p>If you use <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.optim.LBFGS</span></code>, Lightning handles the closure function automatically for you.</p></li>
+<li><p>If you use multiple optimizers, gradients will be calculated only for the parameters of current optimizer
+at each training step.</p></li>
+<li><p>If you need to control how often those optimizers step or override the default <code class="docutils literal notranslate"><span class="pre">.step()</span></code> schedule,
+override the <code class="xref py py-meth docutils literal notranslate"><span class="pre">optimizer_step()</span></code> hook.</p></li>
+</ul>
+</div>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.precision">
+<span class="sig-name descname"><span class="pre">precision</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.precision" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.prepare_data_per_node">
+<span class="sig-name descname"><span class="pre">prepare_data_per_node</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.prepare_data_per_node" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.training">
+<span class="sig-name descname"><span class="pre">training</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.training" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.training_step">
+<span class="sig-name descname"><span class="pre">training_step</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">batch</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="reagent.core.html#reagent.core.types.CBInput" title="reagent.core.types.CBInput"><span class="pre">reagent.core.types.CBInput</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">batch_idx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">optimizer_idx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">0</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.training_step" title="Permalink to this definition"></a></dt>
+<dd><p>Here you compute and return the training loss and some additional metrics for e.g.
+the progress bar or logger.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>batch</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> | (<code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code>, …) | [<code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code>, …]) – The output of your <code class="xref py py-class docutils literal notranslate"><span class="pre">DataLoader</span></code>. A tensor, tuple or list.</p></li>
+<li><p><strong>batch_idx</strong> (<code class="docutils literal notranslate"><span class="pre">int</span></code>) – Integer displaying index of this batch</p></li>
+<li><p><strong>optimizer_idx</strong> (<code class="docutils literal notranslate"><span class="pre">int</span></code>) – When using multiple optimizers, this argument will also be present.</p></li>
+<li><p><strong>hiddens</strong> (<code class="docutils literal notranslate"><span class="pre">Any</span></code>) – Passed in if
+<a href="#id1"><span class="problematic" id="id2">:paramref:`~pytorch_lightning.core.lightning.LightningModule.truncated_bptt_steps`</span></a> &gt; 0.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>Any of.</p>
+<ul class="simple">
+<li><p><code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> - The loss tensor</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">dict</span></code> - A dictionary. Can include any keys, but must include the key <code class="docutils literal notranslate"><span class="pre">'loss'</span></code></p></li>
+<li><dl class="simple">
+<dt><code class="docutils literal notranslate"><span class="pre">None</span></code> - Training will skip to the next batch. This is only for automatic optimization.</dt><dd><p>This is not supported for multi-GPU, TPU, IPU, or DeepSpeed.</p>
+</dd>
+</dl>
+</li>
+</ul>
+</p>
+</dd>
+</dl>
+<p>In this step you’d normally do the forward pass and calculate the loss for a batch.
+You can also do fancier things like multiple forward passes or something model specific.</p>
+<p>Example:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">):</span>
+    <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">z</span> <span class="o">=</span> <span class="n">batch</span>
+    <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+    <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">loss</span>
+</pre></div>
+</div>
+<p>If you define multiple optimizers, this step will be called with an additional
+<code class="docutils literal notranslate"><span class="pre">optimizer_idx</span></code> parameter.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Multiple optimizers (e.g.: GANs)</span>
+<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">optimizer_idx</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="c1"># do training_step with encoder</span>
+        <span class="o">...</span>
+    <span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+        <span class="c1"># do training_step with decoder</span>
+        <span class="o">...</span>
+</pre></div>
+</div>
+<p>If you add truncated back propagation through time you will also get an additional
+argument with the hidden states of the previous step.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Truncated back-propagation through time</span>
+<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">):</span>
+    <span class="c1"># hiddens are the hidden states from the previous truncated backprop step</span>
+    <span class="n">out</span><span class="p">,</span> <span class="n">hiddens</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">lstm</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">)</span>
+    <span class="n">loss</span> <span class="o">=</span> <span class="o">...</span>
+    <span class="k">return</span> <span class="p">{</span><span class="s2">&quot;loss&quot;</span><span class="p">:</span> <span class="n">loss</span><span class="p">,</span> <span class="s2">&quot;hiddens&quot;</span><span class="p">:</span> <span class="n">hiddens</span><span class="p">}</span>
+</pre></div>
+</div>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>The loss value shown in the progress bar is smoothed (averaged) over the last values,
+so it differs from the actual loss returned in train/validation step.</p>
+</div>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.update_params">
+<span class="sig-name descname"><span class="pre">update_params</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">torch.Tensor</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">y</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">torch.Tensor</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">weight</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">torch.Tensor</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.update_params" title="Permalink to this definition"></a></dt>
+<dd><dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>x</strong> – 2D tensor of shape (batch_size, dim)</p></li>
+<li><p><strong>y</strong> – 2D tensor of shape (batch_size, 1)</p></li>
+<li><p><strong>weight</strong> – 2D tensor of shape (batch_size, 1)</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py attribute">
+<dt class="sig sig-object py" id="reagent.training.cb.linucb_trainer.LinUCBTrainer.use_amp">
+<span class="sig-name descname"><span class="pre">use_amp</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">bool</span></em><a class="headerlink" href="#reagent.training.cb.linucb_trainer.LinUCBTrainer.use_amp" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+</section>
+<section id="module-reagent.training.cb">
+<span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-reagent.training.cb" title="Permalink to this headline"></a></h2>
+</section>
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="reagent.training.html" class="btn btn-neutral float-left" title="reagent.training package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="reagent.training.cfeval.html" class="btn btn-neutral float-right" title="reagent.training.cfeval package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2022, Meta Platforms, Inc.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>