From b43ceca1fe83458e2f7ea1cf8d9c447cddb7f202 Mon Sep 17 00:00:00 2001 From: Kaiwen Wang Date: Tue, 22 Dec 2020 23:24:41 -0800 Subject: [PATCH] Replace TableSpec internals with pvc.HiveDataset Summary: 1. Replace namespace, table and ds in TableSpec with pvc.HiveDataset 2. Whenever TableSpec is initialized, we initialize a HiveDataset and pass it in so that it's possible to designate multiple partitions. 3. For pvc queries, use table_spec.dataset directly for most of the time. 4. Update all related interfaces to comply with this change. It turns out TableSpec is defined as a very low level data structure and referenced by many files (>30)... I tried my best to update all of them via code search, unit test and integration test. But to be honest I don't have any context about this rl project at all except knowing this is a fblearner flow pipeline. So please let me know if I miss anything. Thanks! Reviewed By: kittipatv Differential Revision: D25538742 fbshipit-source-id: 5df4d36e60d5717c3042a9728daafa86e080a9da --- reagent/test/workflow/test_preprocessing.py | 3 ++- reagent/test/workflow/test_query_data.py | 3 ++- reagent/test/workflow/test_query_data_parametric.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/reagent/test/workflow/test_preprocessing.py b/reagent/test/workflow/test_preprocessing.py index fdcaab95..f04d920a 100644 --- a/reagent/test/workflow/test_preprocessing.py +++ b/reagent/test/workflow/test_preprocessing.py @@ -5,6 +5,7 @@ import logging import unittest import numpy as np +import pvc import pytest from reagent.preprocessing.identify_types import CONTINUOUS @@ -48,7 +49,7 @@ class TestPreprocessing(ReagentSQLTestBase): num_samples = NUM_ROWS // 2 preprocessing_options = PreprocessingOptions(num_samples=num_samples) - table_spec = TableSpec(table_name=TABLE_NAME) + table_spec = TableSpec(dataset=pvc.HiveDataset(table=TABLE_NAME)) normalization_params = identify_normalization_parameters( table_spec, COL_NAME, preprocessing_options, seed=self.test_class_seed diff --git a/reagent/test/workflow/test_query_data.py b/reagent/test/workflow/test_query_data.py index 2ac6ee09..c68f7522 100644 --- a/reagent/test/workflow/test_query_data.py +++ b/reagent/test/workflow/test_query_data.py @@ -5,6 +5,7 @@ import logging import unittest import numpy as np +import pvc import pytest # pyre-fixme[21]: Could not find `pyspark`. @@ -49,7 +50,7 @@ class TestQueryData(ReagentSQLTestBase): def _discrete_read_data( self, custom_reward_expression=None, gamma=None, multi_steps=None ): - ts = TableSpec(table_name=self.table_name) + ts = TableSpec(dataset=pvc.HiveDataset(table=self.table_name)) dataset: Dataset = query_data( input_table_spec=ts, discrete_action=True, diff --git a/reagent/test/workflow/test_query_data_parametric.py b/reagent/test/workflow/test_query_data_parametric.py index 536bfd77..d43f200d 100644 --- a/reagent/test/workflow/test_query_data_parametric.py +++ b/reagent/test/workflow/test_query_data_parametric.py @@ -5,6 +5,7 @@ import logging import unittest import numpy as np +import pvc import pytest # pyre-fixme[21]: Could not find `pyspark`. @@ -49,7 +50,7 @@ class TestQueryDataParametric(ReagentSQLTestBase): def _parametric_read_data( self, custom_reward_expression=None, gamma=None, multi_steps=None ): - ts = TableSpec(table_name=self.table_name) + ts = TableSpec(dataset=pvc.HiveDataset(table=self.table_name)) dataset: Dataset = query_data( input_table_spec=ts, discrete_action=False,