diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 833268ab..d9d5f1d1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -22,6 +22,9 @@ jobs: - name: Sphinx build run: | sphinx-build docs build-docs + - name: Add CNAME + run: | + echo "docs.malevich.ai" > build-docs/CNAME - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/docs' }} diff --git a/MANIFEST.in b/MANIFEST.in index 37bdda2d..61ccde80 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include requirements.txt include VERSION recursive-include . *.yml -recursive-include . *.yaml \ No newline at end of file +recursive-include . *.yaml +recursive-include . malevich/_templates/* \ No newline at end of file diff --git a/VERSION b/VERSION index 53a75d67..0d91a54c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.6 +0.3.0 diff --git a/docs/API/assistants.rst b/docs/API/assistants.rst new file mode 100644 index 00000000..98c5643e --- /dev/null +++ b/docs/API/assistants.rst @@ -0,0 +1,14 @@ +========== +Assistants +========== + +Assistants are static classes that simplifies working with flows and deployments. They can interpret +tasks, authorize and acquire necessary resources automatically. + +.. automodule:: malevich._deploy + + .. autoclass:: malevich._deploy.Core + :members: + + .. autoclass:: malevich._deploy.Space + :members: diff --git a/docs/Apps/index.rst b/docs/Apps/index.rst deleted file mode 100644 index 33a3c43d..00000000 --- a/docs/Apps/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -===== -Apps -===== - -Apps are the main building blocks of the system. They contain -the actual logic of your product. You either can intergrate with -existing apps or build your custom apps. - -Explore the following sections to learn more about apps: - -.. toctree:: - :maxdepth: 3 - - self - What_is_App - What_is_Processor - Building \ No newline at end of file diff --git a/docs/Contributing/index.rst b/docs/Community/Contributing.rst similarity index 88% rename from docs/Contributing/index.rst rename to docs/Community/Contributing.rst index b37e8be2..9ebded8b 100644 --- a/docs/Contributing/index.rst +++ b/docs/Community/Contributing.rst @@ -12,12 +12,24 @@ Documentation If you were reading the documentation, and spotted a mistake, or something that could be improved, please fork the `docs `_ branch of the repository -and create a pull request with your changes to the :code:`dev/unstable` branch. We will review your +and create a pull request with your changes to the :code:`dev/docs` branch. We will review your changes and merge them into the documentation. As part of documentation is generated from docstrings, we also welcome improvements to the docstrings and interfaces that improve developer experience. +The documentation is built using `Sphinx `_. To build the +documentation locally, you will need to install the dependencies from :code:`requirements.dev.txt`, +and then run + +.. code-block:: console + + rm -rf build-docs && sphinx-build docs build-docs + +.. warning:: + + The branch :code:`dev/docs` is not meant to be merged to any of upstream + branches. It is used only for documentation purposes. Git Workflow ------------ @@ -98,8 +110,3 @@ When posting an issue, it is helpful to include: * A minimal working example that reproduces the bug or describes the feature request * The content of ``~/.malevich/logs`` folder (it will contain useful anonymous information about actions you have performed with Malevich) * The content of ``malevich.yaml`` file from the root of your project (if you have one) - - -.. toctree:: - - self \ No newline at end of file diff --git a/docs/Community/index.rst b/docs/Community/index.rst new file mode 100644 index 00000000..1256a827 --- /dev/null +++ b/docs/Community/index.rst @@ -0,0 +1,8 @@ +Community +========= + +We maintain most of our tools as open-source projects. Learn how can you contribute to them: + +.. toctree:: + + Contributing \ No newline at end of file diff --git a/docs/Apps/Building.rst b/docs/SDK/Apps/Building.rst similarity index 97% rename from docs/Apps/Building.rst rename to docs/SDK/Apps/Building.rst index d59c042b..f5b7fbba 100644 --- a/docs/Apps/Building.rst +++ b/docs/SDK/Apps/Building.rst @@ -17,7 +17,7 @@ To successfully build and run your first app, ensure you have the following: Getting Started --------------- -If you are not yet familiar with :doc:`apps ` or :doc:`processors `, please review the corresponding pages first. +If you are not yet familiar with :doc:`apps ` or :doc:`processors `, please review the corresponding pages first. Let's create an app by running the following command: .. code-block:: console @@ -88,6 +88,8 @@ In case you have a private registry, you have to provide your credentials, so th malevich use image example_app /example_app +Read more about :doc:`dependency management ` + Running a Flow -------------- diff --git a/docs/Apps/What_is_App.rst b/docs/SDK/Apps/Introduction.rst similarity index 74% rename from docs/Apps/What_is_App.rst rename to docs/SDK/Apps/Introduction.rst index c626f351..6bfb134c 100644 --- a/docs/Apps/What_is_App.rst +++ b/docs/SDK/Apps/Introduction.rst @@ -1,11 +1,10 @@ ============ -What is App? +Introduction ============ -.. _pogrebnoijak/julius_export_python: https://hub.docker.com/r/pogrebnoijak/julius_export_python -.. _pogrebnoijak/julius_export_python11: https://hub.docker.com/r/pogrebnoijak/julius_export_python11 -.. _pogrebnoijak/julius_export_python_torch: https://hub.docker.com/r/pogrebnoijak/julius_export_python_torch -.. _pogrebnoijak/julius_export_python_ubuntu: https://hub.docker.com/r/pogrebnoijak/julius_export_python_ubuntu +Apps are the main building blocks of the system. They contain +the actual logic of your product. You either can intergrate with +existing apps or build your custom apps. Apps can be understood as an environment consisting of the following objects: @@ -19,12 +18,13 @@ By interconnecting these objects, you can create a pipeline that solves particul Currently, apps resides within Docker images. To ensure seamless integration with Malevich, an image app must meet the following criteria: -1. The image must be based on one of the following **base images**: +1. The image must be based on one of the following `base images `_: - * `pogrebnoijak/julius_export_python`_ - Optimized for Python 3.9 projects. - * `pogrebnoijak/julius_export_python11`_ - Geared towards Python 3.11 projects. - * `pogrebnoijak/julius_export_python_torch`_ - Customized for Python 3.11 projects requiring PyTorch and CUDA. - * `pogrebnoijak/julius_export_python_ubuntu`_ - Perfect for projects based on Ubuntu 20.04. + * ``malevichai/app:python_v0.1`` - Optimized for Python 3.9 projects. + * ``malevichai/app:python11_v0.1`` - Geared towards Python 3.11 projects. + * ``malevichai/app:python-torch_v0.1`` - Customized for Python 3.11 projects requiring PyTorch and CUDA. + * ``malevichai/app:python-ubuntu_v0.1`` - Perfect for projects based on Ubuntu 20.04. Requires to install Python manually. + * ``malevichai/app:dask_v0.1`` - Python environment with `Dask `_ dataframe backend (instead of Pandas). 2. The app's **codebase must** reside within the `./apps` directory **inside the Docker image**. The inner structure of the `./apps` directory does not matter, all Python files will be parsed and all processor functions will be registered automatically regardless of their location. @@ -33,7 +33,7 @@ Here is a Dockerfile template illustrating how to assemble the `Utility ` function. The function allows you to upload tabular data using :class:`pandas.DataFrame` object or :code:`csv` file. The function returns a `traced` object - a special placeholder that dictates the flow execution engine which operations should be performed on -the data. The data is uploaded (or updated) only on :doc:`interpretation ` stage (or even later). +the data. The data is uploaded (or updated) only on :doc:`interpretation ` stage (or even later). .. warning:: @@ -48,15 +48,26 @@ Example: @flow - def my_flow(): - users = collection('Users Collection', df=data) - # or - users = collection('Users Collection', file='users.csv') - - # Operation on users + def my_flow(input_data=None, file_name=None): + # Create a collection either + # using data frame, or file name + if input_data is not None: + users = collection('Users Collection', df=input_data) + elif file_name is not None: + users = collection('Users Collection', file=file_name) + + + # Operation on `users` collection ... return ... + + task_from_data = my_flow(input_data=df) + task_from_file = my_flow(file_name='users.csv') + + # Operations with task + + ... Assets ------ @@ -87,9 +98,3 @@ can be imported and used in the following way: return ... - -.. toctree:: - :hidden: - :maxdepth: 3 - - self \ No newline at end of file diff --git a/docs/SDK/Deployment/Endpoints.rst b/docs/SDK/Deployment/Endpoints.rst new file mode 100644 index 00000000..eabafd75 --- /dev/null +++ b/docs/SDK/Deployment/Endpoints.rst @@ -0,0 +1,134 @@ +============== +User Endpoints +============== + +After logic for your application is defined with :doc:`Malevich flow `, you can easily integrate it to your app without a need of deploying your script anywhere. Endpoints are dedicated URLs that invoke specified logic on our clouds. Explore the feature in action: + + +Define a logic +-------------- + +Let us start with a simple example. We will define a flow that takes some textual input and returns a +rephrased version of it. Define a flow that utilizes OpenAI's GPT-3 to rephrase a given text. + +Install an app to utilize it in your code: + +.. code-block:: console + + malevich install openai + +And then define a flow: + +.. code-block:: python + + from malevich import flow, collection, CoreInterpreter + from malevich.openai import prompt_completion + + @flow + def rewrite_product_description(): + product_description = collection('product-description') + return prompt_completion( + product_description, + model='gpt-3.5-turbo', + openai_api_key=os.getenv('OPENAI_API_KEY'), + system_prompt=""" + You are SEO specialist. + """, + user_prompt=""" + Rewrite a raw product + description for it to inscrease amount of sales. + + {raw_description} + """ + ) + +Publish an endpoint +------------------- + +Now, let's publish an endpoint that will utilize the flow we have just defined. + +.. code-block:: python + + logic = rewrite_product_description() + logic.interpret(CoreInterpreter(core_auth=('example', 'Welcome to Malevich!'))) + endpoint = logic.publish() + +.. note:: + + Endpoints are defined by a unique generated hash. They are invoked by sending a POST request to the following URL: + + ``https://core.malevich.ai/api/v1/endpoints/run/`` + + +Run an endpoint +--------------- + +They can be invoked from any application that can send HTTP requests as well from the object directly. + +.. code-block:: python + + print('Endpoint hash': endpoint.hash) + print('Endpoint URL': endpoint.get_url()) + + print(endpoint.run( + endpoint_override=EndpointOverride( + cfg=UserConfig(rawMapCollections={ + 'product-description': [ + {'raw_description': 'A beautiful ink pen with a comfortable grip and a smooth writing experience.'} + ] + }) + ) + ).results) + + +There is an example of running an endpoint from JavaScript: + +.. code-block:: javascript + + const myHeaders = new Headers(); + myHeaders.append("Content-Type", "application/json"); + + const raw = JSON.stringify({ + "cfg": { + "rawMapCollections": { + "product-description": [ + { + "raw_description": "A beautiful ink pen with a comfortable grip and a smooth writing experience." + } + ] + } + } + }); + + const requestOptions = { + method: "POST", + headers: myHeaders, + body: raw, + redirect: "follow" + }; + + fetch("https://core.malevich.ai/api/v1/endpoints/run/fae7e96f288fcab9b0ff38ebdda57b87f83d66f1a2d2acd2ac39adaf54d2af91", requestOptions) + .then((response) => response.text()) + .then((result) => console.log(result)) + .catch((error) => console.error(error)); + + + +Update an endpoint +------------------ + +You can update already running endpoint. It can be some minor adjustments or a complete change of the logic. To update, +you can run :meth:`publish ` method with ``hash=`` argument. It will update the endpoint under specified hash. + + +.. code-block:: python + + @flow + def new_logic(): + ... + + + task = new_logic() + task.interpret(CoreInterpreter(core_auth=('example', 'Welcome to Malevich!'))) + endpoint = task.publish(hash='...') + diff --git a/docs/SDK/Deployment/WithCode.rst b/docs/SDK/Deployment/WithCode.rst new file mode 100644 index 00000000..fa4a9a35 --- /dev/null +++ b/docs/SDK/Deployment/WithCode.rst @@ -0,0 +1,91 @@ +=========== +Python API +=========== + +Malevich provides a set of tools to make your flows and apps live. The most straightforward way is to +directly use Malevich Python API and run your flows from code. + +Deployment Assistants +---------------------- + +Assistants are a special type of objects included into the package to shorten the path from flow definition to its actual execution. +They take care of automatic authorization and resource management, and also provides additional interfaces to manipulate and control your flows, deployments and accounts. + + +Core Assistant +++++++++++++++ + +:class:`Core ` assistant automatically authorize you on Malevich Core using credentials of your `Space `_ account. + +.. code-block:: python + + @flow + def write_hello_letter(): + company = collection('company_info') + return prompt_completion( + company, + openai_api_key=os.getenv('OPENAI_API_KEY'), + user_prompt=""" + Given a description of the company, write a welcome letter + + {company} + """ + ) + + + if __name__ == '__main__': + deployment = Core( + write_hello_letter, + user='example', # This can be omitted + access_key='Welcome to Malevich!' # if you are logged in + ) + # Preparing task to be executed + deployment.prepare() + + # Run with data + deployment.run(with_logs=True, override={ + 'company_info': table( + ['Malevich AI - Platform for building AI backends'], + columns=['company'] + ) + }) + + # Get results + print(deployment.results()[0].get_df()) + deployment.stop() + # Get online http endpoint for deployment + print(deployment.publish().get_url()) + + +Space Assistant ++++++++++++++++ + +:class:`Space ` assistant also automatically authorize you and either uploads a new version of the flow or patches the previous one. Also, the assistant can attach to existing deployments without specification of ``@flow`` decorated function or supplying a task. This makes you able to create and manipulate tasks without an access to the flow definition. + +.. code-block:: python + + from malevich import Space, table + + if __name__ == '__main__': + deployment = Space( + reverse_id='write_hello_letter' + ) + # Preparing task to be executed + if deployment.get_stage().value in ['interpreted', 'stopped']: + deployment.prepare() + + # Run with data + deployment.run(override={ + 'company_info': table( + ['Malevich AI - Platform for building AI backends'], + columns=['company'] + ) + }) + + # Get results + print(deployment.results()[0].get_df()) + deployment.stop() + # Get online http endpoint for deployment + print(deployment.publish().get_url()) + + diff --git a/docs/Flows/Autoflow.rst b/docs/SDK/Flows/Autoflow.rst similarity index 98% rename from docs/Flows/Autoflow.rst rename to docs/SDK/Flows/Autoflow.rst index fdc3f23f..6831fa63 100644 --- a/docs/Flows/Autoflow.rst +++ b/docs/SDK/Flows/Autoflow.rst @@ -11,7 +11,7 @@ that copies the signature of the processor. This enables two important things: 1. The function stub gives the user a hint of what the processor expects as input and what it returns as output. It also copies docstrings. So function - is self-contained and her purpose is clear. + is self-contained and its purpose is clear. 2. As stub does not have any implementation, you are not required to install any dependencies that are necessary to run the processor. Once you installed :code:`malevich` package you diff --git a/docs/Flows/index.rst b/docs/SDK/Flows/Introduction.rst similarity index 95% rename from docs/Flows/index.rst rename to docs/SDK/Flows/Introduction.rst index 893ef4bb..5beac6db 100644 --- a/docs/Flows/index.rst +++ b/docs/SDK/Flows/Introduction.rst @@ -1,5 +1,5 @@ -Flows -====== +Introduction +============ Apps can be composed into complex flows that solve real-world problems. A flow dictates the data's path through the system and the order in which apps are executed. @@ -62,11 +62,4 @@ To compose and execute this flow, you would write code as follows: So, the flow definition is totally independent from its execution. -Explore subsequent sections to learn how it all works and how you can manage flows in Malevich: - -.. toctree:: - :maxdepth: 2 - - Autoflow - Lifecycle - Results \ No newline at end of file +Explore subsequent sections to learn how it all works and how you can manage flows in Malevich: \ No newline at end of file diff --git a/docs/Flows/Lifecycle.rst b/docs/SDK/Flows/Lifecycle.rst similarity index 96% rename from docs/Flows/Lifecycle.rst rename to docs/SDK/Flows/Lifecycle.rst index a9e7a55d..3ec372df 100644 --- a/docs/Flows/Lifecycle.rst +++ b/docs/SDK/Flows/Lifecycle.rst @@ -100,6 +100,11 @@ It is important to release resources on the platform when you don't need them an task.run() task.stop() +.. warning:: + + Tasks that are not stopped manually and not active for a certain period are subjects to be suspended + automatically. + Results ------- diff --git a/docs/Flows/Results.rst b/docs/SDK/Flows/Results.rst similarity index 100% rename from docs/Flows/Results.rst rename to docs/SDK/Flows/Results.rst diff --git a/docs/SDK/Project/CI/Github_Action.rst b/docs/SDK/Project/CI/Github_Action.rst new file mode 100644 index 00000000..65d95694 --- /dev/null +++ b/docs/SDK/Project/CI/Github_Action.rst @@ -0,0 +1,93 @@ +===== +CI/CD +===== + +Malevich has its own `Github Actions Workflow `_ pipeline for building and delivering apps to Space. + +------------ +Requirements +------------ + +In order to automatically build your apps and upload them to Space: + +1. App folders should be inside the root directory. + +2. Each app should contain ``Dockerfile`` and ``space.yaml`` files. + +For example: + +.. code-block:: + + your_repository/ + ├─ app1/ + ├─ apps/ + ├─ Dockerfile + ├─ space.yaml + ├─ app2/ + ├─ apps/ + ├─ Dockerfile + ├─ space.yaml + ... + +3. You will need the following credentials: + - `Space credentials `_: Username, Token/Password and Organization slug (optional). + - `Github PAT `_. + - Docker registry info: Registry Host URL, Repository ID/Name, Credentials (username and password). + + Full list: +.. code-block:: console + + └─$ malevich ci github init --help + + Usage: malevich ci github init [OPTIONS] + + Initialize the Github CI/CD pipeline + + ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ --interactive --no-interactive Run the initialization wizard │ + │ --repo-name TEXT Github repository name in the format / │ + │ --github-token TEXT Github token with access to the repository │ + │ --space-user TEXT Malevich Space username │ + │ --space-token TEXT Malevich Space token (password) │ + │ --space-url TEXT Malevich Space API URL │ + │ --branch TEXT Branch to setup CI in │ + │ --registry-url TEXT Docker Image Registry URL, for example `public.ecr.aws` or 'cr.yandex' │ + │ --registry-id TEXT Docker Registry ID │ + │ --image-user TEXT Username to access the Docker Image Registry │ + │ --image-token TEXT Password to access the Docker Image Registry │ + │ --org-id TEXT Malevich space organization ID │ + │ --help Show this message and exit. │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + +--- +Run +--- + +After all requirements were satisfied, use ``malevich ci github init`` command to start initialization. + +.. code-block:: console + + └─$ malevich ci github init --interactive + Welcome to the Github CI/CD pipeline initialization wizard! + Enter the Github repository name in the format /: MalevichAI/malevich-example + Enter the Github token with access to the repository (ghp_...): ghp_******* + Enter the username to access Malevich Space (leave empty to use access token instead): user@example.com + Enter the password to access Malevich Space: *** + Enter the host of Malevich Space (https://dev.api.malevich.ai): https://dev.api.malevich.ai + Enter the branch to run the pipeline on (main): main + Enter the Docker Container Registry type [other/ecr/ecr-private/yandex/ghcr]: ghcr + Enter the Docker Container Registry URL: ghcr.io + Enter the Docker Container Registry ID: repo_id + Enter the username to access the Docker Container Registry: username + Enter the password to access the Docker Container Registry: **** + +This command will pass all arguments to your `github repository secrets `_ and create 2 files inside ``.github/workflows`` directory: + +1. ``malevich_ci__.yml``: Auto CI, which will build and push all apps that were changed in the commit which triggered the workflow. +2. ``malevich_ci__manual.yml``: CI, which can be triggered manually. It will build all the apps in the repository. + +---------- +References +---------- + +- Action source code: https://github.com/MalevichAI/auto-ci \ No newline at end of file diff --git a/docs/SDK/Project/Dependencies.rst b/docs/SDK/Project/Dependencies.rst new file mode 100644 index 00000000..7b9bfb0d --- /dev/null +++ b/docs/SDK/Project/Dependencies.rst @@ -0,0 +1,72 @@ +============== +Dependencies +============== + +Malevich provides a package manager service just like ``pip`` or ``npm``. Each :doc:`app ` that can be run +with Malevich can be installed into the current environment and got manifested in the project. + +Once a package is installed, Malevich creates a *stub module* in the ``malevich`` module. The stub +contains Python objects that can be imported and used within :doc:`Flows `. + +---------- +Installers +---------- + +A Malevich depedency can be installed in different ways. Currently, two ways are provided: + +Install from Image +++++++++++++++++++ + +An app can be installed directly from Docker image. The image should be pushed to any of +image repositories including private ones. Image-based apps can be only used when deployed directly on Core. + +To install with image use the following command: + +.. code-block:: console + + malevich use image [IMAGE_AUTH_USER] [IMAGE_AUTH_PASSWORD] + +Image installer does not require any authorization besides credentials required to access Docker image (in case the image is pushed to a private repository). + +Install from Space ++++++++++++++++++++ + +.. important:: + + Installation using Space requires the project to be connected to a `Space `_ account. + To connect use the following command: + + .. code-block:: console + + malevich space login + + Apps installed with Space can be deployed both on Core directly or via Space + +To install with Space, use the following command: + +.. code-block:: console + + malevich use space [OPTIONS] PACKAGE_NAME REVERSE_ID [BRANCH] [VERSION] + + +Automatic Installation +++++++++++++++++++++++ + +Malevich provides an extensive library of public components that can be installed in your environment. The easiest +way to install a public component is to use ``malevich install `` command: + +.. code-block:: console + + malevich install utility openai + +By default, Space installer is utilized. To use the image installer, use: + +.. code-block:: console + + malevich install utility openai --using image + +Python Environment +++++++++++++++++++ + +Malevich is not managing Python environment. It installs apps into the activated Python environment. Once the environment is changed, you may restore dependencies stated in manifest with ``malevich restore``. To show a list of installed apps, you may use +``malevich list``. \ No newline at end of file diff --git a/docs/SDK/Project/Structure.rst b/docs/SDK/Project/Structure.rst new file mode 100644 index 00000000..58e208cb --- /dev/null +++ b/docs/SDK/Project/Structure.rst @@ -0,0 +1,142 @@ +================= +Managing Projects +================= + +A folder with a file ``malevich.yaml`` is considered to be a Malevich project. Each project can have: + +* Its own set of dependencies +* A set of apps + +Projects can be shared and their dependencies can be easily restored with Malevich CLI. + +---------------------------------- +Project Initialization +---------------------------------- + +Starting a new project on Malevich is done with running the following command: + +.. code-block:: console + + malevich init + + +It will create two files in the your working directory: + +* ``malevich.yaml`` -- a list of installed dependencies and user preferences called *manifest*. +* ``malevich.secrets.yaml`` -- a list of secrets - passwords, access keys and etc. **Make sure that you are not sharing this file or making it public.** + + +----------- +Space Login +----------- + +Usually, project is connected to a particular Space user. To connect a project, run the following command: + +.. code-block:: console + + malevich space login + + +You will be prompted to type your username, password and organization slug. If you do not have password, you should type one of your access tokens. + + +----------------- +Managing Manifest +----------------- + +You can manualy manage your manifest + +.. code-block:: console + + Usage: malevich manifest [OPTIONS] COMMAND [ARGS]... + + Manage the manifest file (malevich.yaml) + + ╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────╮ + │ --help Show this message and exit. │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ╭─ Commands ─────────────────────────────────────────────────────────────────────────────────────────╮ + │ query Query the path in manifest │ + │ secrets Manage secrets stored in manifest │ + │ show Show manifest file │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Manifest works as a multi-level YAML document. Each value in the manifest can +be retrieved by a path. For example, if the manifest is the following: + +.. code-block:: yaml + + dependencies: + - utility: + installer: image + options: + core_auth_token: null + core_auth_user: null + core_host: https://core.malevich.ai/ + image_auth_pass: null + image_auth_user: null + image_ref: public.ecr.aws/o1z1g3t0/utility:latest + package_id: utility + version: '' + preferences: + log_format: RICH + log_level: INFO + verbosity: + Installation: 0 + Intrepretation: 0 + Preparation: 1 + Removal: 0 + Results: 1 + Run: 1 + Stop: 0 + project_id: null + space: null + version: null + +You can retrieve information about ``utility`` by using the following command: + +.. code-block:: console + + malevich manifest query dependencies utility + +or you may get deeper values + +.. code-block:: console + + malevich manifest query dependencies utility options image_ref + + +.. note:: + + As you may notice, ``dependencies`` is the list, but there is no numerical + index in the query. This is due to the structure of the manifest: all lists + contain dictionaries that consist of exactly one key. Thus, each item can be queried by this only key. + This invariant is preserved for the whole manifest. + + +---------------- +Managing Secrets +---------------- + +Secrets are automatically added each time a sensitive piece of information such as passwords appear in the manifest. +Secrets are referenced by their ID as text literals in the form: ``secret#000000``. When a slice of manifest is requested +you may pass ``--resolve-secrets`` flag to substitute all secret literals with actual values. Do this with a caution. + +Restoring secrets ++++++++++++++++++ + +You may want to share a project with other people. However, they may not have access to the secret values you have +used in the project. Once they receive a manifest, they may run the following command to restore missing secrets: + +.. code-block:: console + + malevich secrets restore + +By running the command, the manifest is scanned for secrets and then user is prompted to restore their values. + + +System Project +++++++++++++++ + +If no project is initialized, Malevich uses a global manifest located in ``~/.malevich`` directory. \ No newline at end of file diff --git a/docs/SDK/index.rst b/docs/SDK/index.rst new file mode 100644 index 00000000..190c62cf --- /dev/null +++ b/docs/SDK/index.rst @@ -0,0 +1,40 @@ +User Guide +========== + + +.. toctree:: + :caption: Projects and Environment + + Structure + Dependencies + Automation + +.. toctree:: + :caption: Working with Data + + Introduction + + +.. toctree:: + :caption: Building Apps + + Apps/Introduction + Apps/Processors + Apps/Building + + +.. toctree:: + :caption: Composing Flows + + Flows/Introduction + Flows/Autoflow + Flows/Lifecycle + Flows/Results + + + +.. toctree:: + :caption: Deployment + + Deployment/WithCode + Deployment/Endpoints diff --git a/docs/_static/logo-dark.svg b/docs/_static/logo-dark.svg new file mode 100644 index 00000000..69b2ab9d --- /dev/null +++ b/docs/_static/logo-dark.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/docs/_static/logo-light.svg b/docs/_static/logo-light.svg new file mode 100644 index 00000000..342aa53a --- /dev/null +++ b/docs/_static/logo-light.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/docs/_static/style.css b/docs/_static/style.css index 00a18c90..1c75fe08 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -1,3 +1,16 @@ .wy-nav-content { max-width: 1200px !important; +} +html[data-theme="light"] { + --pst-color-secondary: #A23373; + --pst-color-primary: #753A88; +} + +html[data-theme="dark"] { + --pst-color-secondary: #d85d84; + --pst-color-primary: #AD70C2; +} + +.navbar-brand .logo { + padding: 10px 15px; } \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 03770a53..0561b2e7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,3 +51,9 @@ mermaid_verbose = True autodoc_member_order = 'bysource' autosectionlabel_prefix_document = True +html_theme_options = { + "logo": { + "image_light": "_static/logo-light.svg", + "image_dark": "_static/logo-dark.svg", + } +} diff --git a/docs/index.rst b/docs/index.rst index bd02e622..5fec9956 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,9 +7,9 @@ Getting Started Welcome to `Malevich `_ — a platform for building ML-driven prototypes and iterating them to production. This page provides a brief overview of the platform's capabilities that can be utilized from Python code or the command-line interface. -Explore more about :doc:`building apps ` and :doc:`assembling flows ` to start developing on Malevich. Check out the :doc:`API reference ` for detailed insights into code functionalities. +Explore more about :doc:`building apps ` and :doc:`assembling flows ` to start developing on Malevich. Check out the :doc:`API reference ` for detailed insights into code functionalities. -If you wish to contribute to the Malevich package, please refer to the :doc:`Contributing ` page. +If you wish to contribute to the Malevich package, please refer to the :doc:`Contributing ` page. Installation =============== @@ -105,10 +105,8 @@ Once the apps are installed, you can begin integrating them into a flow. Create # to generate text based on our prompt and extracted names. return text, prompt_completion( entities, - config={ - 'user_prompt': prompt, - 'openai_api_key': os.getenv('OPENAI_API_KEY'), - } + openai_api_key=os.getenv('OPENAI_API_KEY'), + user_prompt=prompt ) if __name__ == '__main__': @@ -125,10 +123,14 @@ Once the apps are installed, you can begin integrating them into a flow. Create ) ) + # Prepare task. + pipeline.prepare() + # Execute the task. - text, brief = pipeline() + pipeline.run() # Save results. + text, brief = pipeline.results() text.get_df().to_csv('text.csv') brief.get_df().to_csv('brief.csv') @@ -144,7 +146,7 @@ Make Your Own Apps ================== We are continually expanding our list of available apps. If you find something missing that you need, we provide -all the tools to create your own apps and optionally share them with the community. See :doc:`Apps/Building` for more details. +all the tools to create your own apps and optionally share them with the community. See :doc:`/SDK/Apps/Building` for more details. .. toctree:: @@ -153,8 +155,6 @@ all the tools to create your own apps and optionally share them with the communi :caption: Contents: Getting Started - Data/index - Apps/index - Flows/index + SDK/index API/index - Contributing/index \ No newline at end of file + Community/index diff --git a/malevich/_deploy.py b/malevich/_deploy.py index 58488219..a9da661f 100644 --- a/malevich/_deploy.py +++ b/malevich/_deploy.py @@ -71,11 +71,12 @@ def __new__( class Space: - + def __new__( cls, task: PromisedTask | FlowFunction[..., Any] | Any = None, # noqa: ANN401, for IDE hints version_mode: VersionMode = VersionMode.MINOR, + reverse_id: str | None = None, force_attach: bool = False, deployment_id: str | None = None, attach_to_last: bool | None = None, @@ -83,7 +84,7 @@ def __new__( *task_args, **task_kwargs ) -> SpaceTask: - + setup = None if not ops: try: @@ -109,9 +110,10 @@ def __new__( if isinstance(task, FlowFunction): task: PromisedTask = task(*task_args, **task_kwargs) - reverse_id = task._component.reverse_id + if isinstance(task, PromisedTask): + reverse_id = task._component.reverse_id - if force_attach: + if force_attach or task is None: return interpreter.attach( reverse_id=reverse_id, deployment_id=deployment_id, diff --git a/malevich/_meta/decor.py b/malevich/_meta/decor.py index a980eb54..2d9ace5e 100644 --- a/malevich/_meta/decor.py +++ b/malevich/_meta/decor.py @@ -4,6 +4,7 @@ from .._autoflow.function import autotrace, sinktrace from ..constants import reserved_config_fields +from ..models.type_annotations import ConfigArgument FnArgs = ParamSpec("FnArgs") FnReturn = TypeVar("FnReturn") @@ -32,6 +33,7 @@ def __init__( for key, value in self.__fn.__annotations__.items() if hasattr(value, '__metadata__') and (metadata := getattr(value, '__metadata__')) + and isinstance(metadata[0], ConfigArgument) and metadata[0].required ] @@ -55,7 +57,7 @@ def _fn_call(self, *args: FnArgs.args, **kwargs: FnArgs.kwargs) -> FnReturn: for reserved, _ in reserved_config_fields: extra_fields.pop(reserved, None) - kwargs['config'] = { + kwargs = { **kwargs['config'], **extra_fields, } diff --git a/malevich/_templates/Dockerfile.app b/malevich/_templates/Dockerfile.app index 3a48a65a..286d5717 100644 --- a/malevich/_templates/Dockerfile.app +++ b/malevich/_templates/Dockerfile.app @@ -6,7 +6,7 @@ # Keep in mind, that source code containing # Malevich-specific code (declaration of processors, inits, etc.) # should be placed into ./apps directory -FROM pogrebnoijak/julius_export_python:0.1 +FROM malevichai/app:python_v0.1 # Copying requirements.txt and installing dependencies COPY requirements.txt requirements.txt diff --git a/malevich/cli.py b/malevich/cli.py index 0d043a57..219ad850 100644 --- a/malevich/cli.py +++ b/malevich/cli.py @@ -74,7 +74,7 @@ app.registered_commands.append( typer.models.CommandInfo( name="new", - help=help.list_["--help"], + help=help.new["--help"], callback=new, cls=typer.core.TyperCommand ) @@ -141,7 +141,7 @@ app.add_typer(dev_app, name='dev') #malevich core -app.add_typer(core_app, name='core') +app.add_typer(core_app, name='core', help=help.core['--help']) # _________________________________________________ diff --git a/malevich/help.py b/malevich/help.py index 76e62e10..3347e9bf 100644 --- a/malevich/help.py +++ b/malevich/help.py @@ -88,3 +88,7 @@ new = { '--help': """Create a new app from a template""", } + +core = { + '--help': """Manage your Core account""" +} diff --git a/malevich/interpreter/core.py b/malevich/interpreter/core.py index 1998cfc5..92e12a5a 100644 --- a/malevich/interpreter/core.py +++ b/malevich/interpreter/core.py @@ -224,7 +224,7 @@ def create_dependency( ) -> CoreInterpreterState: state.depends[caller.owner.uuid].append((callee.owner, link)) _log( - f"Dependency: {caller.owner.short_info()} -> {callee.owner.short_info()}, " + f"Dependency: {callee.owner.short_info()} -> {caller.owner.short_info()}, " f"Link: {link.name}", -1, 0, True ) return state diff --git a/malevich/models/task/interpreted/core.py b/malevich/models/task/interpreted/core.py index de098969..440ba9f1 100644 --- a/malevich/models/task/interpreted/core.py +++ b/malevich/models/task/interpreted/core.py @@ -599,7 +599,7 @@ def publish( **kwargs ) -> MetaEndpoint: from malevich_coretools import create_endpoint, update_endpoint - if self.get_stage() != CoreTaskStage.BUILT: + if self.get_stage() not in [CoreTaskStage.BUILT, CoreTaskStage.ONLINE]: self.prepare(stage=PrepareStages.BUILD) cfg = deepcopy(self.state.cfg) diff --git a/malevich/models/task/promised.py b/malevich/models/task/promised.py index e5c15f11..5c58d773 100644 --- a/malevich/models/task/promised.py +++ b/malevich/models/task/promised.py @@ -329,6 +329,16 @@ def get_interpreted_task(self) -> BaseTask: return self.__task def publish(self, *args, **kwargs) -> MetaEndpoint: + """Creates a HTTP endpoint for the task + + Accepts any arguments and keyword arguments and passes them to the + underlying callback created in the interpreter itself. For particular + arguments and keyword arguments, see the documentation of the interpreter + used before calling this method. + + Returns: + :class:`malevich.models.endpoint.MetaEndpoint`: An endpoint object + """ if not self.__task: raise Exception( "Cannot publish uninterpreted task. " diff --git a/malevich/square/jls.py b/malevich/square/jls.py index 2e4f0fe5..72dc3659 100644 --- a/malevich/square/jls.py +++ b/malevich/square/jls.py @@ -23,7 +23,7 @@ def processor(id: Optional[str] = None, finish_msg: Optional[str] = None, drop_i """Denotes a processor of the app. Processors are core logical units of the app. To - understand processors more, see `What is Processor? <../Apps/What_is_Processor.html>`_ + understand processors more, see :doc:`What is Processor? ` Args: id: The id of the processor. If not provided, the name of the function will be used. diff --git a/malevich/testing/env/__init__.py b/malevich/testing/env/__init__.py new file mode 100644 index 00000000..ff9cfe0e --- /dev/null +++ b/malevich/testing/env/__init__.py @@ -0,0 +1,149 @@ +import json +import os +import logging +from datetime import datetime +from pydantic import BaseModel +from malevich._utility.singleton import SingletonMeta +from malevich._utility.package import package_manager +from ...install.installer import Installer +from malevich.manifest import ManifestManager +from malevich.models.dependency import Dependency +from malevich.install.image import ImageInstaller +from malevich.install.space import SpaceInstaller +from malevich._utility.cache.manager import CacheManager + +TEST_DIR = os.getenv("MALEVICH_TEST_DIR", '~/.malevich.test') +TEST_DIR = os.path.expanduser(TEST_DIR) +os.makedirs(TEST_DIR, exist_ok=True) +env_manf = ManifestManager(TEST_DIR) +cache = CacheManager() + +class _OffloadedDependency(BaseModel): + """When dependency is offloaded, it is saved to this model""" + orig_stub_path: str + offloaded_stub_path: str + dependency: Dependency | None = None + +class EnvManager(metaclass=SingletonMeta): + session = datetime.now().strftime('%d.%m.%y__%H:%M:%S') + installers: dict[str, Installer] = { + 'image': ImageInstaller(), + 'space': SpaceInstaller() + } + def _setup_logger(self) -> None: + self._logger = logging.getLogger('malevich.testing') + self._logger.handlers.clear() + formatter = logging.Formatter( + "%(asctime)s -- %(funcName)s:%(lineno)d -- %(message)s" + ) + handler = logging.FileHandler( + cache.testing.probe_new_entry('protocol.log', entry_group=self.session)[-1], + mode='a+' + ) + handler.setFormatter(formatter) + self._logger.addHandler(handler) + + def __init__(self) -> None: + self._offloaded: list[_OffloadedDependency] = [] + self._logger = None + self._setup_logger() + + + def offload_stub( # noqa: ANN201 + self, + stub_id: str, + dont_fail: bool = True + ): + stub_path = package_manager.get_package_path(stub_id) + if os.path.exists(stub_path): + off_path = cache.testing.cache_dir( + stub_path, entry_group='offload' + ) + return _OffloadedDependency( + orig_stub_path=stub_path, + offloaded_stub_path=off_path, + ) + elif not dont_fail: + raise Exception(f"No stub found for {stub_id}") + + def offload_manf( + self, + package_name: str, + dont_fail: bool = False + ) -> None: + try: + env_manf.remove('dependencies', package_name) + except Exception: + if not dont_fail: + raise + + + + def current_env(self) -> tuple[dict[str, Dependency], list[str]]: + manifest_keys = [ + next(iter(key.keys())) + for key in env_manf.query('dependencies') + ] + manifested_raw = [ + env_manf.query('dependencies', key) + for key in manifest_keys + ] + manifested: dict[str, Dependency] = { + key: ( + self.installers[x['installer']] + .construct_dependency(x) + ) + for key, x in zip( + manifest_keys, + manifested_raw + ) + } + stubs = package_manager.get_all_packages() + return manifested, stubs + + def install(self, dependency: Dependency) -> None: + installer = dependency.installer + self.installers[installer].restore(dependency) + env_manf.put( + 'dependencies', + dependency.package_id, + value={dependency.package_id: dependency.model_dump()}, + append=True + ) + + + def request_env( + self, + dependencies: list[Dependency | tuple[str, Dependency]] + ) -> None: + manifested, _ = self.current_env() + for record in dependencies: + if isinstance(record, tuple): + package_name, dependency = record + else: + package_name = record.package_id + dependency = record + + package_id = dependency.package_id + to_offload_ = None + for manifest_name, manifest_dependency in manifested.items(): + if dependency.compatible_with(manifest_dependency): + print(f"Compatible {package_id}") + break + if manifest_name == package_name: + # Found collision in manifest: should be offloaded + to_offload_ = manifest_name, manifest_dependency + else: + # If no compatible dependency is found, offload the stub + off_ = self.offload_stub(package_id) + if to_offload_: + off_.dependency = to_offload_[-1] + self.offload_manf(to_offload_[0]) + self._offloaded.append(off_) + + # And install our dependency + self.install(dependency) + self._logger.info( + f"install {package_id}, dependency= " + + json.dumps(dependency.model_dump(), separators=(',', ':')) + ) diff --git a/malevich/testing/fixtures.py b/malevich/testing/fixtures.py new file mode 100644 index 00000000..e69de29b