In addition to the system’s built-in content moderation types, Dify also supports user-defined content moderation rules. This method is suitable for developers customizing their own private deployments. For instance, in an enterprise internal customer service setup, it may be required that users, while querying or customer service agents while responding, not only avoid entering words related to violence, sex, and illegal activities but also avoid specific terms forbidden by the enterprise or violating internally established moderation logic. Developers can extend custom content moderation rules at the code level in a private deployment of Dify.
cloud_service.py code template where you can implement specific business logic.
Note: The class variable name must be the same as the custom type name, matching the directory and file names, and must be unique.
from core.moderation.base import Moderation, ModerationAction, ModerationInputsResult, ModerationOutputsResultclass CloudServiceModeration(Moderation): """ The name of custom type must be unique, keep the same with directory and file name. """ name: str = "cloud_service" @classmethod def validate_config(cls, tenant_id: str, config: dict) -> None: """ schema.json validation. It will be called when user saves the config. Example: .. code-block:: python config = { "cloud_provider": "GoogleCloud", "api_endpoint": "https://api.example.com", "api_keys": "123456", "inputs_config": { "enabled": True, "preset_response": "Your content violates our usage policy. Please revise and try again." }, "outputs_config": { "enabled": True, "preset_response": "Your content violates our usage policy. Please revise and try again." } } :param tenant_id: the id of workspace :param config: the variables of form config :return: """ cls._validate_inputs_and_outputs_config(config, True) if not config.get("cloud_provider"): raise ValueError("cloud_provider is required") if not config.get("api_endpoint"): raise ValueError("api_endpoint is required") if not config.get("api_keys"): raise ValueError("api_keys is required") def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult: """ Moderation for inputs. :param inputs: user inputs :param query: the query of chat app, there is empty if is completion app :return: the moderation result """ flagged = False preset_response = "" if self.config['inputs_config']['enabled']: preset_response = self.config['inputs_config']['preset_response'] if query: inputs['query__'] = query flagged = self._is_violated(inputs) # return ModerationInputsResult(flagged=flagged, action=ModerationAction.overridden, inputs=inputs, query=query) return ModerationInputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response) def moderation_for_outputs(self, text: str) -> ModerationOutputsResult: """ Moderation for outputs. :param text: the text of LLM response :return: the moderation result """ flagged = False preset_response = "" if self.config['outputs_config']['enabled']: preset_response = self.config['outputs_config']['preset_response'] flagged = self._is_violated({'text': text}) # return ModerationOutputsResult(flagged=flagged, action=ModerationAction.overridden, text=text) return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response) def _is_violated(self, inputs: dict): """ The main logic of moderation. :param inputs: :return: the moderation result """ return False
At this point, you can select the custom Cloud Service content moderation extension type for debugging in the Dify application orchestration interface.
from core.moderation.base import Moderation, ModerationAction, ModerationInputsResult, ModerationOutputsResultclass CloudServiceModeration(Moderation): """ The name of custom type must be unique, keep the same with directory and file name. """ name: str = "cloud_service" @classmethod def validate_config(cls, tenant_id: str, config: dict) -> None: """ schema.json validation. It will be called when user saves the config. :param tenant_id: the id of workspace :param config: the variables of form config :return: """ cls._validate_inputs_and_outputs_config(config, True) # implement your own logic here def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult: """ Moderation for inputs. :param inputs: user inputs :param query: the query of chat app, there is empty if is completion app :return: the moderation result """ flagged = False preset_response = "" # implement your own logic here # return ModerationInputsResult(flagged=flagged, action=ModerationAction.overridden, inputs=inputs, query=query) return ModerationInputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response) def moderation_for_outputs(self, text: str) -> ModerationOutputsResult: """ Moderation for outputs. :param text: the text of LLM response :return: the moderation result """ flagged = False preset_response = "" # implement your own logic here # return ModerationOutputsResult(flagged=flagged, action=ModerationAction.overridden, text=text) return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
query: the current input content of the end user in a conversation, a fixed parameter for conversational applications.
ModerationInputsResult
flagged: whether it violates the moderation rules
action: action to be taken
direct_output: directly output the preset response
overridden: override the passed variable values
preset_response: preset response (returned only when action=direct_output)
inputs: values passed by the end user, with key as the variable name and value as the variable value (returned only when action=overridden)
query: overridden current input content of the end user in a conversation, a fixed parameter for conversational applications (returned only when action=overridden)
