# -----------------------------------------------------------------------------------------
# (C) Copyright IBM Corp. 2023-2025.
# https://opensource.org/licenses/BSD-3-Clause
# -----------------------------------------------------------------------------------------
from __future__ import annotations
from typing import Any, TYPE_CHECKING, cast, Literal
from tabulate import tabulate
import copy
import logging
from ibm_watsonx_ai.href_definitions import (
API_VERSION,
PIPELINES,
EXPERIMENTS,
SPACES,
RUNTIMES,
)
from ibm_watsonx_ai.wml_client_error import WMLClientError
from ibm_watsonx_ai.wml_resource import WMLResource
if TYPE_CHECKING:
from ibm_watsonx_ai import APIClient
logger = logging.getLogger(__name__)
class MetaProp:
def __init__(
self,
name: str,
key: str,
prop_type: Any,
required: bool,
example_value: Any,
ignored: bool = False,
hidden: bool = False,
default_value: Any = "",
schema: Any = "",
path: str | None = None,
transform: Any = lambda x, client: x,
):
self.key = key
self.name = name
self.prop_type = prop_type
self.required = required
self.example_value = example_value
self.ignored = ignored
self.hidden = hidden
self.schema = schema
self.default_value = default_value
self.path = path if path is not None else "/" + key
self.transform = transform
class MetaNamesBase:
OUTPUT_DATA_SCHEMA = "outputDataSchema"
INPUT_DATA_SCHEMA = "inputDataSchema"
DESCRIPTION = "description"
LABEL_FIELD = "label_column"
TRANSFORMED_LABEL_FIELD = "transformed_label"
def __init__(self, meta_props_definitions: list[MetaProp]) -> None:
self._meta_props_definitions = meta_props_definitions
def _validate(self, meta_props: dict[str, Any]) -> None:
for meta_prop in self._meta_props_definitions:
if (
meta_prop.key != MetaNamesBase.OUTPUT_DATA_SCHEMA
and meta_prop.key != MetaNamesBase.INPUT_DATA_SCHEMA
):
if meta_prop.ignored is False:
WMLResource._validate_meta_prop(
meta_props,
meta_prop.key,
meta_prop.prop_type,
meta_prop.required,
)
else:
if meta_prop.key in meta_props:
logger.warning(
"'{}' meta prop is deprecated. It will be ignored.".format(
meta_prop.name
)
)
def _check_types_only(self, meta_props: dict[str, Any]) -> None:
for meta_prop in self._meta_props_definitions:
if meta_prop.ignored is False:
WMLResource._validate_meta_prop(
meta_props, meta_prop.key, meta_prop.prop_type, False
)
else:
if meta_prop.key in meta_props:
logger.warning(
"'{}' meta prop is deprecated. It will be ignored.".format(
meta_prop.name
)
)
def get(self) -> list[str]:
return sorted(
list(
map(
lambda x: x.name,
filter(
lambda x: not x.ignored and not x.hidden,
self._meta_props_definitions,
),
)
)
)
def show(self) -> None:
print(self._generate_table())
def _generate_doc_table(self) -> str:
return self._generate_table(
name_label="MetaName",
type_label="Type",
required_label="Required",
default_value_label="Default value",
schema_label="Schema",
example_value_label="Example value",
show_examples=True,
format="grid",
values_format="``{}``",
)
def _generate_doc(self, resource_name: str, note: str | None = None) -> str:
docstring_description = f"""
Set of MetaNames for {resource_name}.
Available MetaNames:
{MetaNamesBase(self._meta_props_definitions)._generate_doc_table()}
"""
if note is not None:
docstring_description += f"""
.. note::
{note}
"""
return docstring_description
def _generate_table(
self,
name_label: str = "META_PROP NAME",
type_label: str = "TYPE",
required_label: str = "REQUIRED",
default_value_label: str = "DEFAULT_VALUE",
example_value_label: str = "EXAMPLE_VALUE",
schema_label: str = "SCHEMA",
show_examples: bool = False,
format: str = "simple",
values_format: str = "{}",
) -> str:
show_defaults = any(
meta_prop.default_value != ""
for meta_prop in filter(
lambda x: not x.ignored and not x.hidden, self._meta_props_definitions
)
)
show_schema = any(
meta_prop.schema != ""
for meta_prop in filter(
lambda x: not x.ignored and not x.hidden, self._meta_props_definitions
)
)
header = [name_label, type_label, required_label]
if show_schema:
header.append(schema_label)
if show_defaults:
header.append(default_value_label)
if show_examples:
header.append(example_value_label)
table_content = []
for meta_prop in filter(
lambda x: not x.ignored and not x.hidden, self._meta_props_definitions
):
row = [
meta_prop.name,
meta_prop.prop_type.__name__,
"Y" if meta_prop.required else "N",
]
if show_schema:
row.append(
values_format.format(meta_prop.schema)
if meta_prop.schema != ""
else ""
)
if show_defaults:
row.append(
values_format.format(meta_prop.default_value)
if meta_prop.default_value != ""
else ""
)
if show_examples:
row.append(
values_format.format(meta_prop.example_value)
if meta_prop.example_value != ""
else ""
)
table_content.append(row)
table = tabulate([header] + table_content, tablefmt=format)
return table
def get_example_values(self) -> dict[str, Any]:
return dict(
(x.key, x.example_value)
for x in filter(
lambda x: not x.ignored and not x.hidden, self._meta_props_definitions
)
)
def _generate_resource_metadata(
self,
meta_props: dict[str, Any],
client: APIClient | None = None,
with_validation: bool = False,
initial_metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
if with_validation:
self._validate(meta_props)
if initial_metadata is None:
metadata = {}
else:
metadata = copy.deepcopy(initial_metadata)
def update_map(m: dict | list, path: list[int | str], el: Any) -> None:
if type(m) is dict:
if len(path) == 1:
m[path[0]] = el
else:
if path[0] not in m:
if type(path[1]) is not int:
m[path[0]] = {}
else:
m[path[0]] = []
update_map(m[path[0]], path[1:], el)
elif type(m) is list:
if len(path) == 1:
if len(m) > len(path):
m[cast(int, path[0])] = el
else:
m.append(el)
else:
if len(m) <= cast(int, path[0]):
m.append({})
update_map(m[cast(int, path[0])], path[1:], el)
else:
raise WMLClientError(
"Unexpected metadata path type: {}".format(type(m))
)
for meta_prop_def in filter(
lambda x: not x.ignored, self._meta_props_definitions
):
if meta_prop_def.key in meta_props:
path: list[int | str] = [
int(p) if p.isdigit() else p
for p in meta_prop_def.path.split("/")[1:]
]
update_map(
metadata,
path,
meta_prop_def.transform(meta_props[meta_prop_def.key], client),
)
return metadata
def _generate_patch_payload(
self,
current_metadata: dict[str, Any],
meta_props: dict[str, Any],
client: APIClient | None = None,
with_validation: bool = False,
) -> list[dict]:
if with_validation:
self._check_types_only(meta_props)
def _generate_patch_payload_simple(
meta_props_: dict[str, Any], current_metadata_: dict[str, Any]
) -> list[dict]:
updated_metadata = self._generate_resource_metadata(
meta_props_, client, False, current_metadata_
)
patch_payload: list[dict] = []
def contained_path(
metadata: dict, path: list[int | str]
) -> list[int | str]:
if path[0] in metadata:
if len(path) == 1:
return [path[0]]
else:
rest_of_path = contained_path(metadata[path[0]], path[1:])
if rest_of_path is None:
return [path[0]]
else:
return [path[0]] + rest_of_path
else:
return []
def get_value(metadata: dict, path: list[int | str]) -> Any:
if len(path) == 1:
return metadata[path[0]]
else:
return get_value(metadata[path[0]], path[1:])
def already_in_payload(path: list[int | str]) -> bool:
return any([el["path"] == path for el in patch_payload])
def update_payload(path: list[int | str]) -> None:
existing_path = contained_path(current_metadata_, path)
if len(existing_path) == len(path):
patch_payload.append(
{
"op": "replace",
"path": "/"
+ "/".join([cast(str, e) for e in existing_path]),
"value": get_value(updated_metadata, existing_path),
}
)
else:
if not already_in_payload(existing_path):
final_path = existing_path + [path[len(existing_path)]]
patch_payload.append(
{
"op": "add",
"path": "/"
+ "/".join([cast(str, e) for e in final_path]),
"value": get_value(
updated_metadata,
final_path,
),
}
)
for meta_prop_def in filter(
lambda x: not x.ignored, self._meta_props_definitions
):
if meta_prop_def.key in meta_props_:
path: list[int | str] = [
int(p) if p.isdigit() else p
for p in meta_prop_def.path.split("/")[1:]
]
update_payload(path)
return patch_payload
metadata_props = {
m: meta_props[m]
for d in self._meta_props_definitions
for m in meta_props
if d.key == m and d.path.startswith("/metadata/")
}
entity_props = {
m: meta_props[m]
for d in self._meta_props_definitions
for m in meta_props
if d.key == m and not d.path.startswith("/metadata/")
}
res = []
if metadata_props:
res += _generate_patch_payload_simple(metadata_props, current_metadata)
if entity_props:
meta = (
current_metadata["entity"]
if "entity" in current_metadata
else current_metadata
)
meta.update(
current_metadata["metadata"] if "metadata" in current_metadata else {}
)
res += _generate_patch_payload_simple(
entity_props,
meta,
)
return res
class LearningSystemMetaNames(MetaNamesBase):
_COMPUTE_CONFIGURATION_DEFAULT = "k80"
FEEDBACK_DATA_REFERENCE = "feedback_data_reference"
SPARK_REFERENCE = "spark_instance"
MIN_FEEDBACK_DATA_SIZE = "min_feedback_data_size"
AUTO_RETRAIN = "auto_retrain"
AUTO_REDEPLOY = "auto_redeploy"
COMPUTE_CONFIGURATION = "compute_configuration"
TRAINING_RESULTS_REFERENCE = "training_results_reference"
_meta_props_definitions = [
MetaProp(
"FEEDBACK_DATA_REFERENCE",
FEEDBACK_DATA_REFERENCE,
dict,
True,
example_value={},
),
MetaProp("SPARK_REFERENCE", SPARK_REFERENCE, dict, False, example_value={}),
MetaProp(
"MIN_FEEDBACK_DATA_SIZE",
MIN_FEEDBACK_DATA_SIZE,
int,
True,
example_value=100,
),
MetaProp(
"AUTO_RETRAIN", AUTO_RETRAIN, str, True, example_value="conditionally"
),
MetaProp("AUTO_REDEPLOY", AUTO_REDEPLOY, str, True, example_value="always"),
MetaProp(
"COMPUTE_CONFIGURATION",
COMPUTE_CONFIGURATION,
dict,
False,
example_value={"name": _COMPUTE_CONFIGURATION_DEFAULT},
),
MetaProp(
"TRAINING_RESULTS_REFERENCE",
TRAINING_RESULTS_REFERENCE,
dict,
False,
example_value={
"connection": {
"endpoint_url": "https://s3-api.us-geo.objectstorage.softlayer.net",
"access_key_id": "***",
"secret_access_key": "***",
},
"target": {"bucket": "train-data"},
"type": "s3",
},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("learning system")
class RepositoryMemberMetaNames(MetaNamesBase):
IDENTITY = "identity"
ROLE = "role"
IDENTITY_TYPE = "identity_type"
_meta_props_definitions = [
MetaProp(
"IDENTITY",
IDENTITY,
str,
True,
"IBMid-060000123A (service-ID or IAM-userID)",
),
MetaProp("ROLE", ROLE, str, True, "Supported values - Viewer/Editor/Admin"),
MetaProp(
"IDENTITY_USER", IDENTITY_TYPE, str, True, "Supported values - service/user"
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("Member Specs")
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class PayloadLoggingMetaNames(MetaNamesBase):
PAYLOAD_DATA_REFERENCE = "payload_store"
LABELS = "labels"
OUTPUT_DATA_SCHEMA = "output_data_schema"
_meta_props_definitions = [
MetaProp("PAYLOAD_DATA_REFERENCE", PAYLOAD_DATA_REFERENCE, dict, True, {}),
MetaProp("LABELS", LABELS, list, False, ["a", "b", "c"]),
MetaProp("OUTPUT_DATA_SCHEMA", OUTPUT_DATA_SCHEMA, dict, False, {}),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"payload logging system"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class RuntimeMetaNames(MetaNamesBase):
NAME = "name"
DESCRIPTION = "description"
CUSTOM = "custom"
PLATFORM = "platform"
LIBRARIES_UIDS = "libraries_uids"
CONFIGURATION_FILEPATH = "configuration_filepath"
TAGS = "tags"
SPACE_UID = "space"
COMPUTE = "compute"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "runtime_spec_python_3.10"),
MetaProp("DESCRIPTION", DESCRIPTION, str, False, "sample runtime"),
MetaProp(
"PLATFORM",
PLATFORM,
dict,
True,
'{"name":python","version":"3.10")',
schema={"name(required)": "string", "version(required)": "version"},
),
MetaProp(
"LIBRARIES_UIDS",
LIBRARIES_UIDS,
list,
False,
["46dc9cf1-252f-424b-b52d-5cdd9814987f"],
),
MetaProp(
"CONFIGURATION_FILEPATH",
CONFIGURATION_FILEPATH,
str,
False,
"/home/env_config.yaml",
),
MetaProp(
"TAGS",
TAGS,
list,
False,
[
{
"value": "dsx-project.<project-guid>",
"description": "DSX project guid",
}
],
schema=[{"value(required)": "string", "description(optional)": "string"}],
),
MetaProp("CUSTOM", CUSTOM, dict, False, '{"field1": "value1"}'),
MetaProp(
"SPACE_UID",
SPACE_UID,
str,
False,
path="/space/href",
example_value="46dc9cf1-252f-424b-b52d-5cdd9814987f",
transform=lambda x, client: API_VERSION + SPACES + "/" + x,
),
MetaProp(
"COMPUTE",
COMPUTE,
dict,
False,
example_value={"name": "name1", "nodes": 1},
schema={"name(required)": "string", "nodes(optional)": "string"},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("Runtime Specs")
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class LibraryMetaNames(MetaNamesBase):
NAME = "name"
DESCRIPTION = "description"
FILEPATH = "filepath"
VERSION = "version"
PLATFORM = "platform"
TAGS = "tags"
SPACE_UID = "space_uid"
MODEL_DEFINITION = "model_definition"
CUSTOM = "custom"
COMMAND = "command"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "my_lib"),
MetaProp("DESCRIPTION", DESCRIPTION, str, False, "my lib"),
MetaProp(
"PLATFORM",
PLATFORM,
dict,
True,
{"name": "python", "versions": ["3.10"]},
schema={"name(required)": "string", "version(required)": "version"},
),
MetaProp("VERSION", VERSION, str, True, "1.0"),
MetaProp("FILEPATH", FILEPATH, str, True, "/home/user/my_lib_1_0.zip"),
MetaProp(
"TAGS",
TAGS,
dict,
False,
[
{
"value": "dsx-project.<project-guid>",
"description": "DSX project guid",
}
],
schema=[{"value(required)": "string", "description(optional)": "string"}],
),
MetaProp(
"SPACE_UID",
SPACE_UID,
str,
False,
"3c1ce536-20dc-426e-aac7-7284cf3befc6",
path="/space/href",
transform=lambda x, client: API_VERSION + SPACES + "/" + x,
),
MetaProp("MODEL_DEFINITION", MODEL_DEFINITION, bool, False, False),
MetaProp("COMMAND", COMMAND, str, False, "command"),
MetaProp("CUSTOM", CUSTOM, dict, False, {"field1": "value1"}),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("Custom Libraries")
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
## update this later #Todo
class ShinyMetaNames(MetaNamesBase):
NAME = "name"
DESCRIPTION = "description"
SOFTWARE_SPEC_UID = "software_spec_uid"
SOFTWARE_SPEC_ID = "software_spec_uid"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "Shiny App", path="/metadata/name"),
MetaProp(
"DESCRIPTION",
DESCRIPTION,
str,
False,
"my_description",
path="/metadata/description",
),
MetaProp(
"SOFTWARE_SPEC_UID",
SOFTWARE_SPEC_UID,
str,
False,
"42c36a39-fcc1-5117-8ff6-1d4523e0d6a6",
hidden=True,
),
MetaProp(
"SOFTWARE_SPEC_ID",
SOFTWARE_SPEC_ID,
str,
False,
"42c36a39-fcc1-5117-8ff6-1d4523e0d6a6",
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Shiny Specifications"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class RemoteTrainingSystemMetaNames(MetaNamesBase):
TAGS = "tags"
# SPACE_ID = "space_id"
# PROJECT_ID = "project_id"
NAME = "name"
DESCRIPTION = "description"
CUSTOM = "custom"
ORGANIZATION = "organization"
ALLOWED_IDENTITIES = "allowed_identities"
REMOTE_ADMIN = "remote_admin"
DATA_HANDLER = "data_handler"
LOCAL_TRAINING = "local_training"
HYPERPARAMS = "hyperparams"
MODEL = "model"
_meta_props_definitions = [
MetaProp("TAGS", TAGS, list, False, ["string1", "string2"], schema=["string"]),
# MetaProp('SPACE_ID', SPACE_ID, str, False, '3fc54cf1-252f-424b-b52d-5cdd9814987f', schema=u'string'),
# MetaProp('PROJECT_ID', PROJECT_ID, str, False, '4fc54cf1-252f-424b-b52d-5cdd9814987f', schema=u'string'),
MetaProp("NAME", NAME, str, False, "my-resource"),
MetaProp(
"DESCRIPTION", DESCRIPTION, str, False, "my-resource", schema="string"
),
MetaProp(
"CUSTOM", CUSTOM, dict, False, example_value={"custom_data": "custome_data"}
),
MetaProp(
"ORGANIZATION",
ORGANIZATION,
dict,
False,
example_value={"name": "name", "region": "EU"},
),
MetaProp(
"ALLOWED_IDENTITIES",
ALLOWED_IDENTITIES,
list,
False,
example_value=[{"id": "43689024", "type": "user"}],
),
MetaProp(
"REMOTE_ADMIN",
REMOTE_ADMIN,
dict,
False,
example_value={"id": "id", "type": "user"},
),
MetaProp(
"DATA_HANDLER",
DATA_HANDLER,
dict,
False,
example_value={
"info": {"npz_file": "./data_party0.npz"},
"name": "MnistTFDataHandler",
"path": "mnist_keras_data_handler",
},
),
MetaProp(
"LOCAL_TRAINING",
LOCAL_TRAINING,
dict,
False,
example_value={
"name": "LocalTrainingHandler",
"path": "ibmfl.party.training.local_training_handler",
},
),
MetaProp(
"HYPERPARAMS",
HYPERPARAMS,
dict,
False,
example_value={"epochs": 3, "batch_size": 128},
),
MetaProp(
"MODEL",
MODEL,
dict,
False,
example_value={"info": {"gpu": {"selection": "auto"}}},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Remote Training System"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class ExportMetaNames(MetaNamesBase):
NAME = "name"
DESCRIPTION = "description"
ALL_ASSETS = "all_assets"
ASSET_TYPES = "asset_types"
ASSET_IDS = "asset_ids"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "my-resource"),
MetaProp(
"DESCRIPTION", DESCRIPTION, str, False, "my-resource", schema="string"
),
MetaProp("ALL_ASSETS", ALL_ASSETS, bool, False, False),
MetaProp("ASSET_TYPES", ASSET_TYPES, list, False, example_value=["wml_model"]),
MetaProp(
"ASSET_IDS",
ASSET_IDS,
list,
False,
example_value=[
"13a53931-a8c0-4c2f-8319-c793155e7517",
"13a53931-a8c0-4c2f-8319-c793155e7518",
],
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Export Import metanames"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class VolumeMetaNames(MetaNamesBase):
NAME = "name"
NAMESPACE = "namespace"
STORAGE_CLASS = "storageClass"
STORAGE_SIZE = "storageSize"
EXISTING_PVC_NAME = "existing_pvc_name"
# MOUNT_PATH = "Mountpath"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "my-volume"),
MetaProp("NAMESPACE", NAMESPACE, str, True, "my-volume", schema="string"),
MetaProp(
"STORAGE_CLASS",
STORAGE_CLASS,
str,
False,
example_value="nfs-client",
schema="string",
),
MetaProp("STORAGE_SIZE", STORAGE_SIZE, str, False, example_value="2G"),
# MetaProp('MOUNT_PATH', MOUNT_PATH,str, False, schema=u'string',example_value=""),
MetaProp(
"EXISTING_PVC_NAME",
EXISTING_PVC_NAME,
str,
False,
example_value="volumes-wml-test-input-2-pvc",
schema="string",
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("Volume metanames")
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class GenChatParamsMetaNames(MetaNamesBase):
FREQUENCY_PENALTY = "frequency_penalty"
PRESENCE_PENALTY = "presence_penalty"
TEMPERATURE = "temperature"
MAX_TOKENS = "max_tokens"
TIME_LIMIT = "time_limit"
TOP_P = "top_p"
N = "n"
LOGPROBS = "logprobs"
TOP_LOGPROBS = "top_logprobs"
RESPONSE_FORMAT = "response_format"
_meta_props_definitions = [
MetaProp("FREQUENCY_PENALTY", FREQUENCY_PENALTY, float, False, 1),
MetaProp("PRESENCE_PENALTY", PRESENCE_PENALTY, float, False, 1),
MetaProp("TEMPERATURE", TEMPERATURE, float, False, 0.5),
MetaProp("MAX_TOKENS", MAX_TOKENS, int, False, 100),
MetaProp("TIME_LIMIT", TIME_LIMIT, int, False, 100),
MetaProp("TOP_P", TOP_P, float, False, 0.5),
MetaProp("N", N, int, False, 1),
MetaProp("LOGPROBS", LOGPROBS, bool, False, True),
MetaProp("TOP_LOGPROBS", TOP_LOGPROBS, int, False, 1),
MetaProp(
"RESPONSE_FORMAT", RESPONSE_FORMAT, dict, False, {"type": "json_object"}
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Foundation Model Chat Parameters"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
[docs]
class GenTextParamsMetaNames(MetaNamesBase):
DECODING_METHOD = "decoding_method"
LENGTH_PENALTY = "length_penalty"
TEMPERATURE = "temperature"
TOP_P = "top_p"
TOP_K = "top_k"
RANDOM_SEED = "random_seed"
REPETITION_PENALTY = "repetition_penalty"
MIN_NEW_TOKENS = "min_new_tokens"
MAX_NEW_TOKENS = "max_new_tokens"
STOP_SEQUENCES = "stop_sequences"
TIME_LIMIT = "time_limit"
TRUNCATE_INPUT_TOKENS = "truncate_input_tokens"
RETURN_OPTIONS = "return_options"
PROMPT_VARIABLES = "prompt_variables"
_meta_props_definitions = [
MetaProp("DECODING_METHOD", DECODING_METHOD, str, False, "sample"),
MetaProp(
"LENGTH_PENALTY",
LENGTH_PENALTY,
dict,
False,
{"decay_factor": 2.5, "start_index": 5},
),
MetaProp("TEMPERATURE", TEMPERATURE, float, False, 0.5),
MetaProp("TOP_P", TOP_P, float, False, 0.2),
MetaProp("TOP_K", TOP_K, int, False, 1),
MetaProp("RANDOM_SEED", RANDOM_SEED, int, False, 33),
MetaProp("REPETITION_PENALTY", REPETITION_PENALTY, float, False, 2),
MetaProp("MIN_NEW_TOKENS", MIN_NEW_TOKENS, int, False, 50),
MetaProp("MAX_NEW_TOKENS", MAX_NEW_TOKENS, int, False, 200),
MetaProp("STOP_SEQUENCES", STOP_SEQUENCES, list, False, ["fail"]),
MetaProp("TIME_LIMIT", TIME_LIMIT, int, False, 600000),
MetaProp("TRUNCATE_INPUT_TOKENS", TRUNCATE_INPUT_TOKENS, int, False, 200),
MetaProp(
"PROMPT_VARIABLES", PROMPT_VARIABLES, dict, False, {"object": "brain"}
),
MetaProp(
"RETURN_OPTIONS",
RETURN_OPTIONS,
dict,
False,
{
"input_text": True,
"generated_tokens": True,
"input_tokens": True,
"token_logprobs": True,
"token_ranks": False,
"top_n_tokens": False,
},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Foundation Model Parameters"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
[docs]
class EmbedTextParamsMetaNames(MetaNamesBase):
TRUNCATE_INPUT_TOKENS = "truncate_input_tokens"
RETURN_OPTIONS = "return_options"
_meta_props_definitions = [
MetaProp("TRUNCATE_INPUT_TOKENS", TRUNCATE_INPUT_TOKENS, int, False, 2),
MetaProp(
"RETURN_OPTIONS",
RETURN_OPTIONS,
dict[str, bool],
False,
{"input_text": True},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Foundation Model Embeddings Parameters"
)
def __init__(self):
MetaNamesBase.__init__(self, self._meta_props_definitions)
class GenTextModerationsMetaNames(MetaNamesBase):
INPUT = "input"
OUTPUT = "output"
THRESHOLD = "threshold"
MASK = "mask"
_meta_props_definitions = [
MetaProp("INPUT", INPUT, bool, False, False),
MetaProp("OUTPUT", OUTPUT, bool, False, False),
MetaProp("THRESHOLD", THRESHOLD, float, False, 0.5),
MetaProp("MASK", MASK, dict, False, {"remove_entity_value": True}),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Generation Text Moderations Parameters"
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
[docs]
class GenTextReturnOptMetaNames(MetaNamesBase):
INPUT_TEXT = "input_text"
GENERATED_TOKENS = "generated_tokens"
INPUT_TOKENS = "input_tokens"
TOKEN_LOGPROBS = "token_logprobs"
TOKEN_RANKS = "token_ranks"
TOP_N_TOKENS = "top_n_tokens"
_meta_props_definitions = [
MetaProp("INPUT_TEXT", INPUT_TEXT, bool, True, True),
MetaProp("GENERATED_TOKENS", GENERATED_TOKENS, bool, False, True),
MetaProp("INPUT_TOKENS", INPUT_TOKENS, bool, True, True),
MetaProp("TOKEN_LOGPROBS", TOKEN_LOGPROBS, bool, False, True),
MetaProp("TOKEN_RANKS", TOKEN_RANKS, bool, False, True),
MetaProp("TOP_N_TOKENS", TOP_N_TOKENS, int, False, True),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc(
"Foundation Model Parameters",
note="One of these parameters is required: ['INPUT_TEXT', 'INPUT_TOKENS']",
)
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
class RAGOptimizerConfigurationMetaNames(MetaNamesBase):
NAME = "name"
DESCRIPTION = "description"
INPUT_DATA_REFERENCES = "input_data_references"
TEST_DATA_REFERENCES = "test_data_references"
RESULTS_REFERENCE = "results_reference"
VECTOR_STORE_REFERENCES = "vector_store_references"
HARDWARE_SPEC = "hardware_spec"
_meta_props_definitions = [
MetaProp("NAME", NAME, str, True, "AutoAI RAG Optimizer"),
MetaProp("DESCRIPTION", DESCRIPTION, str, False, "Sample description"),
MetaProp(
"INPUT_DATA_REFERENCES",
INPUT_DATA_REFERENCES,
list,
True,
[
{
"id": "training_input_data",
"name": "training_input_data",
"type": "data_asset",
"connection": {},
"location": {
"href": f"/v2/assets/cbc89dee-a087-420c-9b62-a19931fe3950?project_id=h67df788-73f2-46d0-a787-7453085782ht"
},
}
],
),
MetaProp(
"TEST_DATA_REFERENCES",
TEST_DATA_REFERENCES,
list,
False,
[
{
"id": "test_input_data",
"name": "test_input_data",
"type": "data_asset",
"connection": {},
"location": {
"href": f"/v2/assets/cbc89dee-a087-420c-9b62-a19931fe3950?project_id=h67df788-73f2-46d0-a787-7453085782ht"
},
}
],
),
MetaProp(
"RESULTS_REFERENCE",
RESULTS_REFERENCE,
dict,
True,
{"location": {"path": "."}, "type": "container"},
),
MetaProp(
"VECTOR_STORE_REFERENCES",
VECTOR_STORE_REFERENCES,
list,
False,
[
{
"id": "test_vector_store",
"name": "some_new_milvus_conn",
"type": "connection_asset",
"connection": {"id": "hg5d6a92-1331-h642-b314-14b40bfb0hg4"},
"location": {},
}
],
),
MetaProp(
"HARDWARE_SPEC",
HARDWARE_SPEC,
dict,
True,
{"id": "f4c49h5b-b8e4-444c-9j63-8a7h73020h674", "name": "L"},
),
]
__doc__ = MetaNamesBase(_meta_props_definitions)._generate_doc("rag_optimizer")
def __init__(self) -> None:
MetaNamesBase.__init__(self, self._meta_props_definitions)
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4