From f68bdc5bc3b6a71c53501c508eba64f5afbe9d57 Mon Sep 17 00:00:00 2001 From: timatifey Date: Mon, 19 Sep 2022 15:47:22 +0300 Subject: [PATCH] pydivkit pydivkit --- .../generators/python/generator.py | 1 + .../generators/python/python_entities.py | 2 - json-builder/python/.arcignore | 155 ++++ json-builder/python/.gitignore | 156 ++++ json-builder/python/codegen.sh | 13 + json-builder/python/codegen_config.json | 4 + json-builder/python/pydivkit/__init__.py | 1 + json-builder/python/pydivkit/core/__init__.py | 10 + json-builder/python/pydivkit/core/entities.py | 753 ++++++++++++++++++ json-builder/python/pydivkit/core/fields.py | 87 ++ .../python/pydivkit/core/types/__init__.py | 0 .../python/pydivkit/core/types/union.py | 21 + json-builder/python/pyproject.toml | 39 + json-builder/python/setup.py | 30 + 14 files changed, 1270 insertions(+), 2 deletions(-) create mode 100644 json-builder/python/.arcignore create mode 100644 json-builder/python/.gitignore create mode 100755 json-builder/python/codegen.sh create mode 100644 json-builder/python/codegen_config.json create mode 100644 json-builder/python/pydivkit/__init__.py create mode 100644 json-builder/python/pydivkit/core/__init__.py create mode 100644 json-builder/python/pydivkit/core/entities.py create mode 100644 json-builder/python/pydivkit/core/fields.py create mode 100644 json-builder/python/pydivkit/core/types/__init__.py create mode 100644 json-builder/python/pydivkit/core/types/union.py create mode 100644 json-builder/python/pyproject.toml create mode 100644 json-builder/python/setup.py diff --git a/api_generator/api_generator/generators/python/generator.py b/api_generator/api_generator/generators/python/generator.py index 35605a25a..8dc2055fb 100644 --- a/api_generator/api_generator/generators/python/generator.py +++ b/api_generator/api_generator/generators/python/generator.py @@ -172,6 +172,7 @@ class PythonGenerator(Generator): inner_types_decl = Text() inner_types = sorted(filter(lambda t: isinstance(t, decl_type), entity.inner_types), key=sort_predicate) for ind, inner_type in enumerate(inner_types): + update_base(inner_type) inner_types_decl += decl_method(inner_type) if ind != (len(inner_types) - 1): inner_types_decl += EMPTY diff --git a/api_generator/api_generator/generators/python/python_entities.py b/api_generator/api_generator/generators/python/python_entities.py index fc0937e73..3e6bddd73 100644 --- a/api_generator/api_generator/generators/python/python_entities.py +++ b/api_generator/api_generator/generators/python/python_entities.py @@ -95,8 +95,6 @@ class PythonEntity(Entity): for prop in self.properties: prop.__class__ = PythonProperty cast(PythonProperty, prop).update_base() - for inner_type in self.inner_types: - update_base(inner_type) @property def static_properties(self) -> List[Property]: diff --git a/json-builder/python/.arcignore b/json-builder/python/.arcignore new file mode 100644 index 000000000..b143d1273 --- /dev/null +++ b/json-builder/python/.arcignore @@ -0,0 +1,155 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ +.DS_Store + +.div-schema diff --git a/json-builder/python/.gitignore b/json-builder/python/.gitignore new file mode 100644 index 000000000..75f9cd91f --- /dev/null +++ b/json-builder/python/.gitignore @@ -0,0 +1,156 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ +.DS_Store + +.div-schema +pydivkit/div/** diff --git a/json-builder/python/codegen.sh b/json-builder/python/codegen.sh new file mode 100755 index 000000000..2bc0a2bc6 --- /dev/null +++ b/json-builder/python/codegen.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +scriptDir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +config=$scriptDir/codegen_config.json +schemaDir=$scriptDir/../../schema +outputDir=$scriptDir/pydivkit/div + +apiGenDir=$scriptDir/../../api_generator + +cd $apiGenDir +echo Executing api_generator with [config = $config] [schemaDir = $schemaDir] [outputDir = $outputDir] +python3 -m api_generator -c $config -s $schemaDir -o $outputDir diff --git a/json-builder/python/codegen_config.json b/json-builder/python/codegen_config.json new file mode 100644 index 000000000..2faf5f91e --- /dev/null +++ b/json-builder/python/codegen_config.json @@ -0,0 +1,4 @@ +{ + "lang": "python", + "header": "# Generated code. Do not modify.\n\nfrom __future__ import annotations\nfrom pydivkit.core import BaseDiv, Field\nimport enum\nimport typing\n" +} diff --git a/json-builder/python/pydivkit/__init__.py b/json-builder/python/pydivkit/__init__.py new file mode 100644 index 000000000..98be88605 --- /dev/null +++ b/json-builder/python/pydivkit/__init__.py @@ -0,0 +1 @@ +from pydivkit.div import * # noqa diff --git a/json-builder/python/pydivkit/core/__init__.py b/json-builder/python/pydivkit/core/__init__.py new file mode 100644 index 000000000..a67e95f8c --- /dev/null +++ b/json-builder/python/pydivkit/core/__init__.py @@ -0,0 +1,10 @@ +from .entities import BaseDiv, BaseEntity +from .fields import Field, Ref + + +__all__ = ( + "BaseEntity", + "BaseDiv", + "Field", + "Ref", +) diff --git a/json-builder/python/pydivkit/core/entities.py b/json-builder/python/pydivkit/core/entities.py new file mode 100644 index 000000000..4fee187f9 --- /dev/null +++ b/json-builder/python/pydivkit/core/entities.py @@ -0,0 +1,753 @@ +from __future__ import annotations + +import enum +import hashlib +import json +import uuid +from collections import defaultdict +from functools import reduce +from types import MappingProxyType +from typing import ( + Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Type, Union, + get_args, get_origin, get_type_hints, +) + +import inflection + +from .fields import _Field +from .types.union import inject_types + + +TYPE_FIELD = "type" + +# ExcludeFieldsType: str -> ExcludeFieldsType | bool +ExcludeFieldsType = Mapping[str, Any] +SchemaType = Dict[str, Any] + + +def _make_exclude_fields( + exclude_fields: Optional[Sequence[str]] = None, +) -> ExcludeFieldsType: + if not exclude_fields: + return MappingProxyType({}) + exclude: Dict[str, Any] = {} + exclude_nested: Dict[str, List[str]] = defaultdict(list) + for exclude_field in exclude_fields: + if not exclude_field: + raise ValueError("Field name cannot be empty") + field_name, *nested_fields = exclude_field.split(".") + if nested_fields: + exclude_nested[field_name].append(".".join(nested_fields)) + else: + exclude[field_name] = True + for field_name, nested_fields in exclude_nested.items(): + if field_name not in exclude: + exclude[field_name] = _make_exclude_fields(nested_fields) + return MappingProxyType(exclude) + + +def _cast_value_type(value: Any, type_: Any) -> Any: # noqa + if isinstance(value, _Field): + raise ValueError("Value cannot be of type _Field") + + if type_ is Any: + return value + + origin_type = get_origin(type_) + if origin_type is list: + if not isinstance(value, list): + raise ValueError( + f"Value {value} has wrong type. Expected type is {type_}.", + ) + element_type, *_ = get_args(type_) + return [_cast_value_type(v, element_type) for v in value] + + if origin_type is dict: + if not isinstance(value, dict): + raise ValueError( + f"Value {value} has wrong type. Expected type is {type_}.", + ) + + key_type, value_type = get_args(type_) + return { + _cast_value_type(k, key_type): _cast_value_type(v, value_type) + for k, v in value.items() + } + + if origin_type is Union: + if not getattr(type_, "__injected_types__", False): + inject_types(type_) + setattr(type_, "__injected_types__", True) + types = getattr(type_, "__types__", None) + if types and isinstance(value, dict): + type_value = value.get(TYPE_FIELD) + if type_value: + target_type = types.get(type_value) + if target_type: + return target_type(**value) + raise ValueError( + f"Union {type_} does not contain type {type_value}.", + ) + raise ValueError(f"Value {value} does not have field {TYPE_FIELD}.") + for u_type in get_args(type_): + try: + return _cast_value_type(value, u_type) + except ValueError: + pass + raise ValueError( + f"Value {value} has wrong type. Expected type is {type_}.", + ) + + if isinstance(value, type_): + return value + + if isinstance(value, BaseEntity): + if isinstance(value, type_): + return value + raise ValueError( + f"Value {value} has wrong type. Expected type is {type_}", + ) + + if issubclass(type_, BaseEntity) and isinstance(value, dict): + return type_(**value) + + if value is not None: + try: + return type_(value) + except Exception: + pass + + raise ValueError( + f"Value {value} has wrong type. Expected type is {type_}.", + ) + + +def dump(obj: Any) -> Any: + if isinstance(obj, list): + return [dump(obj_item) for obj_item in obj] + elif isinstance(obj, BaseEntity): + return obj.dict() + elif isinstance(obj, enum.Enum): + return obj.value + return obj + + +def _update_related_templates( + value: Any, + related_templates: Set[Type[BaseDiv]], +) -> None: + if isinstance(value, BaseEntity): + related_templates.update(value.related_templates()) + elif isinstance(value, list): + for v_item in value: + _update_related_templates(v_item, related_templates) + + +def _unpack_optional_type(type_: Any) -> Any: + if get_origin(type_) is Union: + inner_types = [] + for inner_type in get_args(type_): + if get_origin(inner_type) or not isinstance(None, inner_type): + inner_types.append(inner_type) + return Union[tuple(inner_types)] + return type_ + + +def _merge_types(fst_type: Any, snd_type: Any) -> Any: + fst_type = _unpack_optional_type(fst_type) + snd_type = _unpack_optional_type(snd_type) + if fst_type == snd_type: + return fst_type + raise TypeError(f"Incompatible types: {fst_type} and {snd_type}") + + +BUILTIN_TYPES_TO_SCHEMA: Mapping[type, Mapping[str, Any]] = MappingProxyType( + { + int: MappingProxyType({"type": "integer"}), + float: MappingProxyType({"type": "number"}), + bool: MappingProxyType( + { + "type": "integer", + "enum": [0, 1], + "format": "boolean", + }, + ), + str: MappingProxyType({"type": "string"}), + }, +) + + +def _type_field_to_schema(field: _Field) -> SchemaType: + return { + "type": "string", + "enum": [field.default], + } + + +def _enum_to_schema(type_: Type[enum.Enum]) -> SchemaType: + return { + "type": "string", + "enum": [enum_el.value for enum_el in type_], + } + + +def _list_to_schema( + type_: Any, + definitions: Dict[str, SchemaType], + exclude: ExcludeFieldsType, +) -> SchemaType: + item_type, *_ = get_args(type_) + return { + "type": "array", + "items": _field_to_schema(None, item_type, exclude, definitions), + } + + +def _dict_to_schema() -> SchemaType: + return { + "type": "object", + "additionalProperties": True, + } + + +def _union_to_schema( + field: Optional[_Field], + type_: Any, + exclude: ExcludeFieldsType, + definitions: Dict[str, SchemaType], +) -> SchemaType: + return { + "anyOf": [ + _field_to_schema(field, arg_type, exclude, definitions) + for arg_type in get_args(type_) + ], + } + + +def _add_field_extra_to_schema(field: _Field, schema: SchemaType) -> None: + if field.description: + schema["description"] = field.description + if field.default: + schema["default"] = field.default + schema.update(**field.constraints) + + +def _field_to_schema( + field: Optional[_Field], + type_: Any, + exclude: ExcludeFieldsType, + definitions: Dict[str, SchemaType], +) -> SchemaType: + type_ = _unpack_optional_type(type_) + origin = get_origin(type_) + + schema: Optional[SchemaType] = None + if field and field.name == TYPE_FIELD and field.default: + schema = _type_field_to_schema(field) + elif origin is list: + schema = _list_to_schema(type_, definitions, exclude) + elif origin is dict: + schema = _dict_to_schema() + elif origin is Union: + schema = _union_to_schema(field, type_, exclude, definitions) + elif type_ in BUILTIN_TYPES_TO_SCHEMA: + schema = {**BUILTIN_TYPES_TO_SCHEMA[type_]} + elif issubclass(type_, BaseEntity): + schema = type_.schema_as_ref(definitions, exclude) + elif issubclass(type_, enum.Enum): + schema = _enum_to_schema(type_) + + if not schema: + raise TypeError(f"Schema building error for unknown type {type_}") + + if field: + _add_field_extra_to_schema(field, schema) + + return schema + + +class BaseEntity: + __fields__: Mapping[str, _Field] + __field_names__: Mapping[uuid.UUID, str] + __field_types__: Mapping[str, type] + + __subclasses__: Dict[str, Type[BaseEntity]] = {} + __related_templates__: FrozenSet[Type[BaseDiv]] = frozenset({}) + + def __init__(self, **kwargs: Any) -> None: + if self.__fields__ and not self.__field_types__: + class_name = self.__class__.__name__ + raise TypeError( + f"Class {class_name} fields are not prepared yet, you may " + f"need to call {class_name}.update_forward_refs().", + ) + self._instance_related_templates: Set[Type[BaseDiv]] = set() + for field_name, value in kwargs.items(): + if value is not None: + setattr(self, field_name, value) + for field_name, field in self.__fields__.items(): + if field_name not in kwargs: + setattr(self, field_name, field.default) + + def __init_subclass__(cls, *args: Any, **kwargs: Any) -> None: + super().__init_subclass__() + if cls.__name__ in cls.__subclasses__: + raise TypeError(f"Component named {cls.__name__} already exists") + cls.__subclasses__[cls.__name__] = cls + cls.__fields__ = cls._extract_fields() + cls.__field_names__ = cls._extract_field_names(cls.__fields__) + cls._remove_fields_from_cls_attrs() + cls.update_forward_refs() + if cls.__init__ != BaseEntity.__init__: + setattr(cls, "__init__", BaseEntity.__init__) + + def __setattr__(self, key: str, value: Any) -> None: + if not key.startswith("_"): + field_type = self.__field_types__.get(key) + if field_type is None: + raise KeyError( + f"'{self.__class__.__name__}' object " + f"has no attribute '{key}'", + ) + if key == TYPE_FIELD: + self._validate_type_value(value) + if not (isinstance(value, _Field) and value.ref_to): + value = _cast_value_type(value, field_type) + _update_related_templates(value, self._instance_related_templates) + super().__setattr__(key, value) + + @classmethod + def _extract_field_names( + cls, + fields: Mapping[str, _Field], + ) -> Mapping[uuid.UUID, str]: + return MappingProxyType( + {field.uid: field_name for field_name, field in fields.items()}, + ) + + @classmethod + def _remove_fields_from_cls_attrs(cls) -> None: + to_remove = [] + for attr_name, attr_value in cls.__dict__.items(): + if isinstance(attr_value, _Field): + to_remove.append(attr_name) + for attr_name in to_remove: + delattr(cls, attr_name) + + def _validate_type_value(self, value: Any) -> None: + field = self.__fields__[TYPE_FIELD] + if field.default and field.default != value: + raise ValueError( + f"Property '{TYPE_FIELD}' has wrong value: '{value}'. " + f"Expected is '{field.default}'.", + ) + + @classmethod + def update_forward_refs(cls) -> None: + try: + cls.__field_types__ = cls._extract_field_types(cls.__fields__) + except (NameError, AttributeError): + cls.__field_types__ = MappingProxyType({}) + + @classmethod + def _extract_fields(cls) -> Mapping[str, _Field]: + fields: Dict[str, _Field] = {} + for base in cls.__bases__: + base_fields = getattr(base, "__fields__", None) + if issubclass(base, BaseEntity) and base_fields: + fields.update(base_fields) + for key, value in cls.__dict__.items(): + if isinstance(value, _Field): + field_ref = value.ref_to + if field_ref: + value = fields[key] + if value.is_ref: + raise ValueError("Ref cannot point to another ref") + if value.default is not None: + raise ValueError( + f"Cannot create a ref for field {key} because " + f"it has a default value", + ) + value.ref_to = field_ref + if not value.name: + value.name = key + fields[key] = value + return MappingProxyType(fields) + + @classmethod + def _extract_field_types( + cls, + fields: Mapping[str, _Field], + ) -> Mapping[str, type]: + cls_hints = get_type_hints(cls, localns={cls.__name__: cls}) + cls._validate_field_types_of_bases(cls_hints) + cls._validate_field_types(fields, cls_hints) + + field_types = {} + for key in fields.keys(): + field_types[key] = cls_hints[key] + for base in cls.__bases__: + base_field_types = getattr(base, "__field_types__", None) + if issubclass(base, BaseEntity) and base_field_types: + field_types.update(base_field_types) + + for field_name, field_type in field_types.items(): + field = cls.__fields__[field_name] + if field.ref_to: + field_types[field_name] = Optional[field_type] + + return MappingProxyType(field_types) + + @classmethod + def _validate_field_types( + cls, + fields: Mapping[str, _Field], + cls_hints: Mapping[str, Any], + ) -> None: + for key, field in fields.items(): + hint = cls_hints.get(key) + if not hint: + raise ValueError( + f"Type hint is missed for {cls.__name__}.{key}", + ) + if field.default is not None: + _cast_value_type(field.default, hint) + + @classmethod + def _validate_field_types_of_bases( + cls, + cls_hints: Mapping[str, Any], + ) -> None: + for base in cls.__bases__: + base_hints = get_type_hints(base) + for key, cls_type_hint in cls_hints.items(): + if not isinstance(getattr(base, key, None), _Field): + continue + base_type_hint = base_hints.get(key) + if base_type_hint and (cls_type_hint is not base_type_hint): + raise ValueError( + f"Type hint mismatch. _Field {cls.__name__}.{key} " + "should match type hint of a parent class " + f"{base.__name__}.{key}\n" + f"Expected: {base_type_hint}. " + f"Actual: {cls_type_hint}", + ) + + @classmethod + def _merge_ref_types( + cls, + *ref_types_seq: Mapping[uuid.UUID, Any], + ) -> Mapping[uuid.UUID, Any]: + all_refs: Dict[uuid.UUID, List[Any]] = defaultdict(list) + for ref_types in ref_types_seq: + for ref_uid, ref_type in ref_types.items(): + all_refs[ref_uid].append(ref_type) + return MappingProxyType( + { + ref_uid: reduce(_merge_types, ref_types) + for ref_uid, ref_types in all_refs.items() + }, + ) + + @classmethod + def _extract_ref_types_from_obj( + cls, + obj: Any, + ) -> Mapping[uuid.UUID, Any]: + if isinstance(obj, BaseEntity): + return obj._extract_all_ref_types() + elif isinstance(obj, list): + ref_types_seq = [] + for item in obj: + if isinstance(item, BaseEntity): + ref_types_seq.append(item._extract_all_ref_types()) + return cls._merge_ref_types(*ref_types_seq) + return MappingProxyType({}) + + def _extract_ref_types_from_values(self) -> Mapping[uuid.UUID, Any]: + ref_types: Dict[uuid.UUID, Any] = {} + for field_name, field_type in self.__field_types__.items(): + field_value = getattr(self, field_name, None) + if isinstance(field_value, _Field) and field_value.ref_to: + field = self.__fields__[field_name] + field_value.ref_to.apply_constraints(field.constraints) + ref_uid = field_value.ref_to.uid + if ref_uid in ref_types: + ref_types[ref_uid] = _merge_types( + ref_types[ref_uid], + field_type, + ) + else: + ref_types[ref_uid] = field_type + return MappingProxyType(ref_types) + + def _extract_all_ref_types(self) -> Mapping[uuid.UUID, Any]: + ref_types_seq = [self._extract_ref_types_from_values()] + for field_name in self.__fields__: + field_value = getattr(self, field_name, None) + ref_types_seq.append(self._extract_ref_types_from_obj(field_value)) + return self._merge_ref_types(*ref_types_seq) + + def related_templates(self) -> Set[Type[BaseDiv]]: + return {*self.__related_templates__, *self._instance_related_templates} + + def dict(self) -> Dict[str, Any]: + result: Dict[str, Any] = {} + for field_name, field in self.__fields__.items(): + field_value = getattr(self, field_name, field.default) + if field_value is not None: + if isinstance(field_value, _Field) and field_value.ref_to: + result[ + f"${field.field_name}" + ] = field_value.ref_to.field_name + else: + result[field.field_name] = dump(field_value) + return result + + @classmethod + def _can_add_field_to_schema( + cls, + field_name: str, + field: _Field, + ) -> bool: + return not field.ref_to + + @classmethod + def _build_schema( + cls, + definitions: Dict[str, SchemaType], + exclude: ExcludeFieldsType, + ) -> SchemaType: + properties: Dict[str, Any] = {} + required: List[str] = [] + for field_name, field in cls.__fields__.items(): + if ( + not cls._can_add_field_to_schema(field_name, field) + or exclude.get(field_name) is True + ): + continue + field_type = cls.__field_types__[field_name] + properties[field_name] = _field_to_schema( + field=field, + type_=field_type, + exclude=exclude.get(field_name, {}), + definitions=definitions, + ) + if field_type != Optional[field_type]: + required.append(field_name) + schema: SchemaType = {"type": "object"} + if properties: + schema["properties"] = properties + if required: + schema["required"] = required + return schema + + @classmethod + def schema_as_ref( + cls, + definitions: Dict[str, SchemaType], + exclude: ExcludeFieldsType, + ) -> SchemaType: + schema_name = cls.__name__ + if exclude: + serialized_exclude = json.dumps(exclude, sort_keys=True) + exclude_hash = hashlib.md5(serialized_exclude.encode()).hexdigest() + schema_name += f"_{exclude_hash}" + if schema_name not in definitions: + definitions[schema_name] = {} # to stop infinity recursion + schema = cls._build_schema(definitions, exclude) + definitions[schema_name] = schema + return {"$ref": f"#/definitions/{schema_name}"} + + +class BaseDiv(BaseEntity): + __tpl_values__: Mapping[str, Any] + __refs__: Mapping[str, Set[Optional[str]]] + __local_referred_fields__: FrozenSet[str] + + __base_type__: Optional[str] = None + __template__: Optional[Dict[str, Any]] = None + + def __init_subclass__(cls, *args: Any, **kwargs: Any) -> None: + cls.__base_type__ = cls._get_base_type() + cls.__local_referred_fields__ = cls._extract_local_referred_fields() + cls.__template__ = None + cls._inject_type_field() + super().__init_subclass__(*args, **kwargs) + + @classmethod + def _extract_local_referred_fields(cls) -> FrozenSet[str]: + return frozenset( + { + field_name + for field_name, field in cls.__dict__.items() + if isinstance(field, _Field) and field.ref_to + }, + ) + + def __setattr__(self, key: str, value: Any) -> None: + if (value is None) and (key in self.__tpl_values__): + return + super().__setattr__(key, value) + + @classmethod + def update_forward_refs(cls) -> None: + super().update_forward_refs() + cls.__tpl_values__ = cls._extract_tpl_values(cls.__field_types__) + cls.__field_types__ = cls._make_fields_optional_from_tpl_values( + cls.__field_types__, + cls.__tpl_values__, + ) + cls.__related_templates__ = cls._extract_related_templates( + cls.__tpl_values__, + ) + cls._validate_ref_types() + + @classmethod + def _make_fields_optional_from_tpl_values( + cls, + field_types: Mapping[str, type], + tpl_values: Mapping[str, Any], + ) -> Mapping[str, type]: + new_field_types: Dict[str, Any] = {**field_types} + for field_name in tpl_values: + new_field_types[field_name] = Optional[new_field_types[field_name]] + return MappingProxyType(new_field_types) + + @classmethod + def _extract_related_templates( + cls, + tpl_values: Mapping[str, Any], + ) -> FrozenSet[Type[BaseDiv]]: + related_templates: Set[Type[BaseDiv]] = set() + if cls.__base_type__: + related_templates.add(cls) + for base_cls in cls.__bases__: + if issubclass(base_cls, BaseEntity): + related_templates.update(base_cls.__related_templates__) + for tpl_value in tpl_values.values(): + _update_related_templates(tpl_value, related_templates) + return frozenset(related_templates) + + @classmethod + def _extract_tpl_values( + cls, + field_types: Mapping[str, Any], + ) -> Mapping[str, Any]: + tpl_values: Dict[str, Any] = {} + for name, field_type in field_types.items(): + if not hasattr(cls, name): + continue + value = getattr(cls, name) + if value is not None: + tpl_values[name] = _cast_value_type(value, field_type) + delattr(cls, name) + return MappingProxyType(tpl_values) + + @classmethod + def _extract_ref_types_from_tpl_values(cls) -> Mapping[uuid.UUID, Any]: + return cls._merge_ref_types( + *( + cls._extract_ref_types_from_obj(tpl_value) + for tpl_value in cls.__tpl_values__.values() + ), + ) + + @classmethod + def _extract_ref_types_from_fields(cls) -> Mapping[uuid.UUID, Any]: + ref_types: Dict[uuid.UUID, Any] = {} + for field_name, field in cls.__fields__.items(): + if not field.ref_to: + continue + field.ref_to.apply_constraints(field.constraints) + ref_uid = field.ref_to.uid + field_type = cls.__field_types__[field_name] + if ref_uid in ref_types: + ref_types[ref_uid] = _merge_types( + ref_types[ref_uid], + field_type, + ) + else: + ref_types[ref_uid] = field_type + return MappingProxyType(ref_types) + + @classmethod + def _validate_ref_types(cls) -> None: + ref_types = cls._merge_ref_types( + cls._extract_ref_types_from_fields(), + cls._extract_ref_types_from_tpl_values(), + ) + for ref_uid, ref_type in ref_types.items(): + field_name = cls.__field_names__[ref_uid] + expected_field_type = cls.__field_types__[field_name] + if ( + ref_type != expected_field_type + and ref_type != Optional[expected_field_type] + ): + raise TypeError( + f"Type of attribute '{field_name}' does " + f"not match ref type {expected_field_type} != {ref_type}", + ) + + @classmethod + def _get_base_type(cls) -> Optional[str]: + if len(cls.__bases__) > 1: + raise TypeError( + "Types conflict: base class cannot be uniquely identified", + ) + base_cls, *_ = cls.__bases__ + if not issubclass(base_cls, BaseDiv): + raise TypeError( + "Types conflict: class must be derived from the BaseDiv class", + ) + type_field = base_cls.__fields__.get(TYPE_FIELD) + if (base_cls is BaseDiv) or not (type_field and type_field.default): + return None + return type_field.default + + @classmethod + def _inject_type_field(cls) -> None: + if cls.__base_type__: + type_value = inflection.underscore(cls.__name__) + setattr(cls, TYPE_FIELD, _Field(default=type_value)) + + @classmethod + def template(cls) -> Dict[str, Any]: + if not cls.__base_type__: + raise TypeError(f"Component {cls.__name__} is not a template") + if cls.__template__ is None: + cls.__template__ = cls._build_template() + return cls.__template__ + + @classmethod + def _build_template(cls) -> Dict[str, Any]: + template = {TYPE_FIELD: cls.__base_type__} + for field_name, tpl_field_value in cls.__tpl_values__.items(): + field = cls.__fields__[field_name] + template[field.field_name] = dump(tpl_field_value) + for field_name, field in cls.__fields__.items(): + if field.ref_to and field_name in cls.__local_referred_fields__: + template[f"${field.field_name}"] = field.ref_to.field_name + return template + + @classmethod + def _can_add_field_to_schema( + cls, + field_name: str, + field: _Field, + ) -> bool: + return ( + super()._can_add_field_to_schema(field_name, field) + and field_name not in cls.__tpl_values__ + ) + + @classmethod + def schema( + cls, + exclude_fields: Optional[Sequence[str]] = None, + ) -> SchemaType: + definitions: Dict[str, SchemaType] = {} + exclude = _make_exclude_fields(exclude_fields) + schema = cls._build_schema(definitions, exclude) + schema["definitions"] = definitions + return schema diff --git a/json-builder/python/pydivkit/core/fields.py b/json-builder/python/pydivkit/core/fields.py new file mode 100644 index 000000000..9f8876ee9 --- /dev/null +++ b/json-builder/python/pydivkit/core/fields.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, Mapping, Optional + + +@dataclass() +class _Field: + uid: uuid.UUID = field(default_factory=uuid.uuid4) + name: Optional[str] = field(default=None) + default: Optional[Any] = field(default=None) + description: Optional[str] = field(default=None) + constraints: Dict[str, Any] = field(default_factory=dict) + ref_to: Optional[_Field] = field(default=None) + is_ref: bool = field(default=False) + + @property + def field_name(self) -> str: + if not self.name: + raise ValueError("Field 'name' not initialized") + return self.name + + def apply_constraints(self, constraints: Mapping[str, Any]) -> None: + for c_key, c_value in constraints.items(): + if ( + self.constraints.get(c_key) + and self.constraints[c_key] != c_value + ): + raise ValueError( + f"Incompatible constraints: {c_key} = " + f"{self.constraints[c_key]} and {c_key} = {c_value}", + ) + self.constraints[c_key] = c_value + + +def _make_constraints(**kwargs: Any) -> Dict[str, Any]: + return {k: v for k, v in kwargs.items() if v is not None} + + +def Field( + name: Optional[str] = None, + description: Optional[str] = None, + default: Optional[Any] = None, + format: Optional[str] = None, + gt: Optional[float] = None, + ge: Optional[float] = None, + lt: Optional[float] = None, + le: Optional[float] = None, + multiple_of: Optional[float] = None, + min_items: Optional[int] = None, + max_items: Optional[int] = None, + unique_items: Optional[bool] = None, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + regex: Optional[str] = None, +) -> Any: + return _Field( + name=name, + description=description, + default=default, + constraints=_make_constraints( + **{ + "format": format, + "exclusiveMinimum": gt, + "minimum": ge, + "exclusiveMaximum": lt, + "maximum": le, + "multipleOf": multiple_of, + "minItems": min_items, + "maxItems": max_items, + "uniqueItems": unique_items, + "minLength": min_length, + "maxLength": max_length, + "regex": regex, + }, + ), + ) + + +def Ref(field: Any) -> Any: + if not isinstance(field, _Field): + raise ValueError( + f"Value {field} has wrong type. Expected type is Field.", + ) + field.is_ref = True + return _Field(ref_to=field) diff --git a/json-builder/python/pydivkit/core/types/__init__.py b/json-builder/python/pydivkit/core/types/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/json-builder/python/pydivkit/core/types/union.py b/json-builder/python/pydivkit/core/types/union.py new file mode 100644 index 000000000..7ce84ae9d --- /dev/null +++ b/json-builder/python/pydivkit/core/types/union.py @@ -0,0 +1,21 @@ +from types import MappingProxyType +from typing import Any, Dict, Type, Union, get_args, get_origin + + +def inject_types(type_: Any) -> None: + if get_origin(type_) is not Union: + return + types_dict: Dict[str, Type[Any]] = {} + for u_type in get_args(type_): # TODO + fields = getattr(u_type, "__fields__", None) + if fields is None: + continue + field = fields.get("type") + if field is None: + continue + type_value = getattr(field, "default", None) + if (type_value is None) or not isinstance(type_value, str): + continue + types_dict[type_value] = u_type + if types_dict: + setattr(type_, "__types__", MappingProxyType(types_dict)) diff --git a/json-builder/python/pyproject.toml b/json-builder/python/pyproject.toml new file mode 100644 index 000000000..f2a69c1ca --- /dev/null +++ b/json-builder/python/pyproject.toml @@ -0,0 +1,39 @@ +[tool.poetry] +name = "pydivkit" +version = "3.0.1" +description = "" +authors = [ + "Vladislav Bakaev ", + "Dmitry Orlov " +] +include = ["pydivkit/div/*"] + +[tool.poetry.dependencies] +python = "^3.8" +inflection = "^0.5.1" + +[tool.poetry.dev-dependencies] +pytest = "^7.0.1" +pytest-asyncio = "^0.18.1" +mypy = "^0.931" +pylama = "^8.3.0" +pre-commit = "^2.17.0" +yesqa = "^1.3.0" +gray = "^0.10.2" +teamcity-messages = "^1.31" + +[tool.mypy] +cache_dir = "/dev/null" +strict = true +ignore_missing_imports = true +allow_subclassing_any = true +allow_untyped_calls = true +pretty = true +show_error_codes = true +implicit_reexport = true +allow_untyped_decorators = true +warn_return_any = false + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/json-builder/python/setup.py b/json-builder/python/setup.py new file mode 100644 index 000000000..83b1c4204 --- /dev/null +++ b/json-builder/python/setup.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from setuptools import setup + +packages = \ +['pydivkit', 'pydivkit.core', 'pydivkit.core.types', 'pydivkit.div'] + +package_data = \ +{'': ['*']} + +install_requires = \ +['inflection>=0.5.1,<0.6.0'] + +setup_kwargs = { + 'name': 'pydivkit', + 'version': '3.0.1', + 'description': '', + 'long_description': None, + 'author': 'Vladislav Bakaev', + 'author_email': 'bakaev-vlad@yandex-team.ru', + 'maintainer': None, + 'maintainer_email': None, + 'url': None, + 'packages': packages, + 'package_data': package_data, + 'install_requires': install_requires, + 'python_requires': '>=3.8,<4.0', +} + + +setup(**setup_kwargs)