diff --git a/smile_talend_job/README.rst b/smile_talend_job/README.rst new file mode 100644 index 000000000..486d9a47b --- /dev/null +++ b/smile_talend_job/README.rst @@ -0,0 +1,116 @@ +===================== +Talend Job Execution +===================== +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-Smile_SA%2Fodoo_addons-lightgray.png?logo=github + :target: https://github.com/Smile-SA/odoo_addons/tree/13.0/smile_talend_job + :alt: Smile-SA/odoo_addons + +|badge2| |badge3| + +This module allows the execution of Talend jobs , it is useful during data integration. + +Features: + +* Execution of a Talend job by importing its archive . +* Visualization of the job's execution logs (creation date, end date, details, state, execution duration). +* Offers the possibility to add context variables to manage various execution types (database to use, username, password, port, host...). +* Definition of children jobs. +* Definition of the parent job. +* Passing a context environment to children jobs. +* Control the allocated memory for the job execution (by specifying the argument in the args field). +* Jobs and their executions logs storage . + + +**Table of contents** + +.. contents:: + :local: + + +Usage +===== + +To run a Talend job: + +#. Go to ``Settings > Talend Jobs`` menu. +#. Press the button ``Create``. +#. Insert the name of the job (the name should be the same name used for the job in Talend studio ). +#. Upload the job's archive file (zip format) to ``archive field`` . +#. Click on ``Run`` button. +#. Click on ``Refresh logs`` button. + +.. figure:: static/description/job_creation.png + :alt: Job creation + :width: 900px + +To propagate a context environment to children jobs : + +#. Add the concerned children jobs to ``Children field`` . +#. Click on ``Propagate context`` button. + +.. figure:: static/description/context_propagate.png + :alt: Propagate Context + :width: 900px + +Parameters you can specify : + +#. Path : to specify the path to the job's directory . +#. Args : add -Xms64M or -Xmx1024M to control the available memory for the job's execution. +#. Loop : to specify the number of times the job will be executed. +#. Parent : to indicate the parent job. +#. Children : to add children jobs. +#. Context : to specify job's context parameters. + +.. figure:: static/description/path.png + :alt: Path Update + :width: 900px + +.. figure:: static/description/args.png + :alt: Allocated memory control + :width: 900px + +.. figure:: static/description/context.png + :alt: Context Update + :width: 900px + + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + +Do not contact contributors directly about support or help with technical issues. + +GDPR / EU Privacy +================= + +This addons does not collect any data and does not set any browser cookies. + +Credits +======= + +Contributors +------------ + +* Corentin POUHET-BRUNERIE + +Maintainer +---------- + +This module is maintained by Smile SA. + +Since 1991 Smile has been a pioneer of technology and also the European expert in open source solutions. + +.. image:: https://avatars0.githubusercontent.com/u/572339?s=200&v=4 + :alt: Smile SA + :target: http://smile.fr + +This module is part of the `odoo-addons `_ project on GitHub. + +You are welcome to contribute. diff --git a/smile_talend_job/__init__.py b/smile_talend_job/__init__.py new file mode 100755 index 000000000..83aed83d6 --- /dev/null +++ b/smile_talend_job/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import models diff --git a/smile_talend_job/__manifest__.py b/smile_talend_job/__manifest__.py new file mode 100644 index 000000000..84e67c51c --- /dev/null +++ b/smile_talend_job/__manifest__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +{ + "name": "Talend Jobs", + "version": "0.1", + "depends": [ + "mail", + ], + "author": "Smile", + "license": 'AGPL-3', + "description": """""", + "summary": "", + "website": "http://www.smile.fr", + "category": 'Tools', + "sequence": 20, + "data": [ + 'security/ir.model.access.csv', + 'views/talend_job_view.xml', + 'views/talend_job_logs_view.xml', + ], + "demo": [ + 'demo/talend_jobs.xml', + ], + "auto_install": True, + "installable": True, + "application": False, +} diff --git a/smile_talend_job/demo/jobInfo_0.1.zip b/smile_talend_job/demo/jobInfo_0.1.zip new file mode 100644 index 000000000..bf2ee647d Binary files /dev/null and b/smile_talend_job/demo/jobInfo_0.1.zip differ diff --git a/smile_talend_job/demo/talend_jobs.xml b/smile_talend_job/demo/talend_jobs.xml new file mode 100644 index 000000000..228bb8fda --- /dev/null +++ b/smile_talend_job/demo/talend_jobs.xml @@ -0,0 +1,11 @@ + + + + + + jobInfo + + + + + diff --git a/smile_talend_job/i18n/fr.po b/smile_talend_job/i18n/fr.po new file mode 100644 index 000000000..eebdaaed3 --- /dev/null +++ b/smile_talend_job/i18n/fr.po @@ -0,0 +1,347 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * smile_talend_job +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 13.0-20180803\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-08-06 07:51+0000\n" +"PO-Revision-Date: 2018-08-06 07:51+0000\n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_needaction +msgid "Action Needed" +msgstr "Actions nécessaires" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__active +msgid "Active" +msgstr "Actif" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__archive +msgid "Archive" +msgstr "Archive" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Archived" +msgstr "Archivé" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__args +msgid "Args" +msgstr "Args" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_attachment_ids +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs_attachment_ids +msgid "Attachments" +msgstr "Pièces jointes" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__child_ids +msgid "Children" +msgstr "Enfants" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__command +msgid "Command" +msgstr "Commande" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__context +msgid "Context" +msgstr "Contexte" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__create_uid +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__create_uid +msgid "Created by" +msgstr "Créé par" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__create_date +msgid "Created on" +msgstr "Créé le" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__logs +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_logs_form +msgid "Details" +msgstr "Détails" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__display_name +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__display_name +msgid "Display Name" +msgstr "Nom à afficher" + +#. module: smile_talend_job +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job__last_log_state__done +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job_logs__state__done +msgid "Done" +msgstr "Terminé" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__end_date +msgid "End Date" +msgstr "Date de fin" + +#. module: smile_talend_job +#: code:addons/PycharmProjects/odoo_addons/smile_talend_job/models/talend_job.py:0 +#: code:addons/smile_talend_job/models/talend_job.py:0 +#, python-format +msgid "Error! You cannot create recursive hierarchy of Talend jobs." +msgstr "Erreur ! Vous ne pouvez pas créer de hiérarchie récursive avec les jobs Talend" + +#. module: smile_talend_job +#: code:addons/PycharmProjects/odoo_addons/smile_talend_job/models/talend_job.py:0 +#: code:addons/smile_talend_job/models/talend_job.py:0 +#, python-format +msgid "Execution already in progress" +msgstr "Exécution déjà en cours" + +#. module: smile_talend_job +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job__last_log_state__failed +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job_logs__state__failed +msgid "Failed" +msgstr "Echoué" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_id +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs_id +msgid "ID" +msgstr "ID" + +#. module: smile_talend_job +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_needaction +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_unread +msgid "If checked, new messages require your attention." +msgstr "En cas de contrôle, les nouveaux messages nécessitent votre attention." + +#. module: smile_talend_job +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_has_error +msgid "If checked, some messages have a delivery error." +msgstr "Si la case est cochée, certains messages contiennent une erreur de livraison." + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_is_follower +msgid "Is Follower" +msgstr "Est un abonné" + +#. module: smile_talend_job +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job__last_log_state__killed +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job_logs__state__killed +msgid "Killed" +msgstr "Tué" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__last_log_date +msgid "Last Execution Date" +msgstr "Date de dernière exécution" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__last_log_state +msgid "Last Execution Status" +msgstr "Etat de la dernière exécution" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job____last_update +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs____last_update +msgid "Last Modified on" +msgstr "Dernière modification le" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__write_uid +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__write_uid +msgid "Last Updated by" +msgstr "Dernière mise à jour par" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__write_date +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__write_date +msgid "Last Updated on" +msgstr "Dernière mise à jour le" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__log_ids +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Logs" +msgstr "Historique" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__loop +msgid "Loop" +msgstr "Boucle" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_main_attachment_id +msgid "Main Attachment" +msgstr "Pièce jointe principale" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_search +msgid "Main Jobs" +msgstr "Jobs maîtres" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_has_error +msgid "Message Delivery error" +msgstr "Erreur de livraison du message" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_ids +msgid "Messages" +msgstr "Messages" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__name +msgid "Name" +msgstr "Nom" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_needaction_counter +msgid "Number of Actions" +msgstr "Nombre d'actions" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_has_error_counter +msgid "Number of errors" +msgstr "Nombre d'erreurs" + +#. module: smile_talend_job +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_needaction_counter +msgid "Number of messages which requires an action" +msgstr "Nombre de messages qui nécessitent une action" + +#. module: smile_talend_job +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_has_error_counter +msgid "Number of messages with delivery error" +msgstr "Nombre de messages avec erreur de livraison" + +#. module: smile_talend_job +#: model:ir.model.fields,help:smile_talend_job.field_talend_job__message_unread_counter +msgid "Number of unread messages" +msgstr "Nombre de messages non lus" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__parent_id +msgid "Parent" +msgstr "Parent" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__path +msgid "Path" +msgstr "Chemin" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__sequence +msgid "Priority" +msgstr "Priorité" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__pid +msgid "Process Id" +msgstr "Id du processus" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Propagate context" +msgstr "Propager le contexte" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Refresh logs" +msgstr "Rafraîchir les logs" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Run" +msgstr "Exécuter" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Run only it" +msgstr "Exécuter uniquement celui-ci" + +#. module: smile_talend_job +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job__last_log_state__running +#: model:ir.model.fields.selection,name:smile_talend_job.selection__talend_job_logs__state__running +msgid "Running" +msgstr "En cours" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__create_date +msgid "Start Date" +msgstr "Date de début" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__state +msgid "Status" +msgstr "État" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_logs_form +msgid "Stop" +msgstr "Arrêter" + +#. module: smile_talend_job +#: model:ir.model,name:smile_talend_job.model_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__job_id +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "Talend Job" +msgstr "Job Talend" + +#. module: smile_talend_job +#: model:ir.model,name:smile_talend_job.model_talend_job_logs +msgid "Talend Job Logs" +msgstr "Historique des jobs Talend" + +#. module: smile_talend_job +#: model:ir.actions.act_window,name:smile_talend_job.action_talend_jobs +#: model:ir.ui.menu,name:smile_talend_job.menu_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_search +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_tree +msgid "Talend Jobs" +msgstr "Jobs Talend" + +#. module: smile_talend_job +#: code:addons/PycharmProjects/odoo_addons/smile_talend_job/models/talend_job.py:0 +#: code:addons/smile_talend_job/models/talend_job.py:0 +#, python-format +msgid "This module support only zipfiles" +msgstr "Ce module supporte uniquement les fichiers ZIP" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__time +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job_logs__time_human +msgid "Time" +msgstr "Durée" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_unread +msgid "Unread Messages" +msgstr "Nombre de messages non lus" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__message_unread_counter +msgid "Unread Messages Counter" +msgstr "Compteur de messages non lus" + +#. module: smile_talend_job +#: model:ir.model.fields,field_description:smile_talend_job.field_talend_job__version +msgid "Version" +msgstr "Version" + +#. module: smile_talend_job +#: model_terms:ir.ui.view,arch_db:smile_talend_job.view_talend_job_form +msgid "database=test" +msgstr "database=test" diff --git a/smile_talend_job/models/__init__.py b/smile_talend_job/models/__init__.py new file mode 100755 index 000000000..bd8899e90 --- /dev/null +++ b/smile_talend_job/models/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import talend_job +from . import talend_job_logs diff --git a/smile_talend_job/models/talend_job.py b/smile_talend_job/models/talend_job.py new file mode 100644 index 000000000..5e4f714ed --- /dev/null +++ b/smile_talend_job/models/talend_job.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 +import binascii +import codecs +import csv +import io +import logging +import os +import stat +import sys +import tempfile +import threading +import zipfile + +from odoo import api, fields, models, registry, _ +from odoo.exceptions import UserError, ValidationError + +from .talend_job_logs import STATES + + +_logger = logging.getLogger(__name__) + + +class ExecutionError(Exception): + pass + + +class TalendJob(models.Model): + _name = 'talend.job' + _description = 'Talend Job' + _inherit = 'mail.thread' + _order = 'sequence, id' + + name = fields.Char(required=True) + active = fields.Boolean(default=True) + sequence = fields.Integer('Priority', required=True, default=15) + archive = fields.Binary() + context = fields.Text() + path = fields.Char() + args = fields.Char() + log_ids = fields.One2many( + 'talend.job.logs', 'job_id', 'Logs', readonly=True, copy=False) + parent_id = fields.Many2one('talend.job', 'Parent') + child_ids = fields.One2many( + 'talend.job', 'parent_id', 'Children', copy=False) + version = fields.Char(compute='_get_job_version', store=True) + command = fields.Char(compute='_get_command') + loop = fields.Integer(required=True, default=1) + last_log_date = fields.Datetime( + 'Last Execution Date', compute='_get_last_log_infos') + last_log_state = fields.Selection( + STATES, 'Last Execution Status', compute='_get_last_log_infos') + + @api.constrains('parent_id') + def _check_hierarchy(self): + if not self._check_recursion(): + raise ValidationError(_('Error! You cannot create ' + 'recursive hierarchy of Talend jobs.')) + + @api.depends('archive', 'child_ids.archive') + def _get_job_version(self): + for record in self: + if record.archive: + with record._get_zipfile() as zf: + filename = 'jobInfo.properties' + # INFO: can't use configparser because this file has no section + with zf.open(filename) as f: + reader = csv.reader( + codecs.iterdecode(f.readlines(), 'utf-8'), + delimiter='=', escapechar='\\', quoting=csv.QUOTE_NONE) + for row in reader: + if row[0] == 'jobVersion': + record.version = row[1] + else: + record.version = max( + record._get_all_children().filtered('version').mapped('version'), + default='') + + def _get_all_children(self): + all_children = children = self.mapped('child_ids') + while children.mapped('child_ids'): + children = children.mapped('child_ids') + all_children |= children + return all_children + + @api.depends('log_ids.state') + def _get_last_log_infos(self): + for record in self: + not_killed_logs = record.log_ids.filtered( + lambda log: log.state != 'killed') + if not_killed_logs: + last_log = not_killed_logs.sorted('create_date')[-1] + record.last_log_date = last_log.create_date + record.last_log_state = last_log.state + else: + record.last_log_date = False + record.last_log_state = False + + @api.depends('name', 'path', 'args', 'context') + def _get_command(self): + for record in self: + cmd = [record._get_exefile()] + if record.args: + cmd += record.args.split(' ') + context = record.context and \ + record.context.replace(' ', '').replace('\t', '') or '' + for param in filter(bool, context.split('\n')): + cmd += ['--context_param', param] + record.command = ' '.join(cmd) + + def run(self): + return self._run() + + def run_only(self): + return self._run(depth=0) + + def _run(self, depth=-1): + queue = [] + self._build_queue(queue, depth) + if self._context.get('in_new_thread', True): + thread = threading.Thread( + target=self._process_queue, args=(queue,)) + thread.start() + else: + self._process_queue(queue) + return True + + def _build_queue(self, queue, depth=-1): + self._check_execution() + for job in self: + if job.archive: + queue.append((job.id, depth)) + job.child_ids._build_queue(queue, depth) + + def _check_execution(self): + if self.mapped('log_ids').filtered(lambda log: log.state == 'running'): + raise UserError(_('Execution already in progress')) + + @api.model + def _process_queue(self, queue): + while queue: + job_id, depth = queue.pop(0) + try: + self._process_job(job_id) + except ExecutionError: + pass + else: + self._add_child_jobs(job_id, depth, queue) + + @api.model + def _process_job(self, job_id): + with api.Environment.manage(): + with registry(self._cr.dbname).cursor() as auto_cr: + # autocommit: each insert/update request + # will be performed atomically. Thus + # everyone (with another cursor) can access to + # a running Talend job logs + self = self.with_env(self.env(cr=auto_cr)) + self._cr.autocommit(True) + log = self.env['talend.job.logs'].create({'job_id': job_id}) + + if log.state != 'done': + msg = 'JOB failed'.format( + job_id, self.browse(job_id).name) + raise ExecutionError(msg) + + @api.model + def _add_child_jobs(self, job_id, depth, queue): + if depth: + depth -= 1 + with api.Environment.manage(): + with registry(self._cr.dbname).cursor() as auto_cr: + self = self.with_env(self.env(cr=auto_cr)) + for child in self.browse(job_id).child_ids: + queue.append((child.id, depth)) + + def _get_zipfile(self): + self.ensure_one() + data = self.archive + try: + data = base64.b64decode(data) + except binascii.Error: + pass + f = io.BytesIO(data) + if not zipfile.is_zipfile(f): + raise UserError(_('This module support only zipfiles')) + return zipfile.ZipFile(f) + + def _get_path(self): + self.ensure_one() + path = self.path or tempfile.gettempdir() + if not os.path.exists(path): + os.makedirs(path) + return path + + def _get_exefile(self): + ext = 'bat' if sys.platform == 'win32' else 'sh' + return os.path.join( + self._get_path(), self.name, '%s_run.%s' % (self.name, ext)) + + def _get_contextfile(self): + return os.path.join(self._get_path(), 'defaults.properties') + + def _prepare(self): + for rec in self: + with rec._get_zipfile() as zf: + zf.extractall(rec._get_path()) + os.chmod(rec._get_exefile(), + # -rwxr-xr-x + stat.S_IRWXU + + stat.S_IRGRP + stat.S_IXGRP + + stat.S_IROTH + stat.S_IXOTH) + + def refresh_logs(self): + return True + + def propagate_context(self): + for job in self: + children = job.with_context(active_test=False)._get_all_children() + children.write({'context': job.context}) + return True diff --git a/smile_talend_job/models/talend_job_logs.py b/smile_talend_job/models/talend_job_logs.py new file mode 100644 index 000000000..74658955d --- /dev/null +++ b/smile_talend_job/models/talend_job_logs.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import logging +import psutil +import shlex +from subprocess import PIPE, Popen +import time + +from odoo import api, fields, models, SUPERUSER_ID +from odoo.modules.registry import Registry +from odoo.tools.func import wraps + +from ..tools import s2human + +_logger = logging.getLogger(__name__) + +STATES = [ + ('running', 'Running'), + ('done', 'Done'), + ('killed', 'Killed'), + ('failed', 'Failed'), +] + + +def state_cleaner(model): + def decorator(method): + @wraps(method) + def wrapper(self, cr, *args, **kwargs): + res = method(self, cr, *args, **kwargs) + env = api.Environment(cr, SUPERUSER_ID, {}) + if model._name in env.registry.models: + Model = env[model._name] + cr.execute("select relname from pg_class " + "where relname='%s'" % model._table) + if cr.rowcount: + Model.search( + [('state', '=', 'running')]).filtered( + lambda rec: not rec.pid or + not psutil.pid_exists(rec.pid)).kill() + return res + return wrapper + return decorator + + +class TalendJobLogs(models.Model): + _name = 'talend.job.logs' + _description = 'Talend Job Logs' + _rec_name = 'create_date' + _order = 'create_date desc' + + def __init__(self, pool, cr): + super(TalendJobLogs, self).__init__(pool, cr) + model = pool[self._name] + if not getattr(model, '_state_cleaner', False): + model._state_cleaner = True + setattr(Registry, 'setup_models', state_cleaner(model)( + getattr(Registry, 'setup_models'))) + + job_id = fields.Many2one( + 'talend.job', 'Talend Job', require=True, ondelete='cascade') + state = fields.Selection( + STATES, 'Status', readonly=True, required=True, default='running') + pid = fields.Integer('Process Id', readonly=True) + logs = fields.Text('Details', readonly=True, default="") + create_date = fields.Datetime('Start Date', readonly=True) + end_date = fields.Datetime(readonly=True) + time = fields.Integer(compute='_get_time') + time_human = fields.Char('Time', compute='_get_time') + + @api.depends('end_date') + def _get_time(self): + for record in self: + to_date = record.end_date or fields.Datetime.now() + timedelta = fields.Datetime.from_string(to_date) \ + - fields.Datetime.from_string(record.create_date) + record.time = timedelta.total_seconds() + record.time_human = s2human(record.time) + + @api.model + def create(self, vals): + log = super(TalendJobLogs, self).create(vals) + log._run_job() + return log + + def _run_job(self): + for rec in self: + try: + rec.job_id._prepare() + loop = rec.job_id.loop + while loop: + rec._execute() + loop -= 1 + except Exception as e: + rec.logs += str(e) + rec.write({ + 'end_date': fields.Datetime.now(), + 'state': 'failed', + }) + else: + rec.write({ + 'end_date': fields.Datetime.now(), + 'state': 'done', + }) + + def _execute(self): + for record in self: + args = shlex.split(record.job_id.command) + proc = Popen(args, stdout=PIPE, stderr=PIPE) + record.pid = proc.pid + while proc.poll() is None: + # INFO: communicate returns (outs, errors) + for index, logs in enumerate(proc.communicate()): + if logs: + logs = logs.decode('utf-8') + record.logs += '%s\n' % logs + if index: + _logger.error(logs) + else: + _logger.info(logs) + time.sleep(1) + + def kill(self): + self.filtered('pid')._kill() + return self.write({'state': 'killed'}) + + def _kill(self): + for record in self: + try: + proc = psutil.Process(record.pid) + proc.kill() + except psutil.NoSuchProcess: + pass + + def unlink(self): + self.filtered(lambda talend_log: talend_log.state == 'running').kill() + return super(TalendJobLogs, self).unlink() diff --git a/smile_talend_job/security/ir.model.access.csv b/smile_talend_job/security/ir.model.access.csv new file mode 100644 index 000000000..25612fd67 --- /dev/null +++ b/smile_talend_job/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +talend_job_system,talend.job,smile_talend_job.model_talend_job,base.group_system,1,1,1,1 +talend_job_logs_system,talend.job.logs,smile_talend_job.model_talend_job_logs,base.group_system,1,1,1,1 diff --git a/smile_talend_job/static/description/args.png b/smile_talend_job/static/description/args.png new file mode 100644 index 000000000..e005bffb2 Binary files /dev/null and b/smile_talend_job/static/description/args.png differ diff --git a/smile_talend_job/static/description/context.png b/smile_talend_job/static/description/context.png new file mode 100644 index 000000000..1c3c3d364 Binary files /dev/null and b/smile_talend_job/static/description/context.png differ diff --git a/smile_talend_job/static/description/context_propagate.png b/smile_talend_job/static/description/context_propagate.png new file mode 100644 index 000000000..f3676e04c Binary files /dev/null and b/smile_talend_job/static/description/context_propagate.png differ diff --git a/smile_talend_job/static/description/icon.png b/smile_talend_job/static/description/icon.png new file mode 100755 index 000000000..17984e2d0 Binary files /dev/null and b/smile_talend_job/static/description/icon.png differ diff --git a/smile_talend_job/static/description/index.html b/smile_talend_job/static/description/index.html new file mode 100644 index 000000000..8ac0ee8f1 --- /dev/null +++ b/smile_talend_job/static/description/index.html @@ -0,0 +1,464 @@ + + + + + + +Talend Job Execution + + + +
+

Talend Job Execution

+ +

License: AGPL-3 Smile-SA/odoo_addons

+

This module allows the execution of Talend jobs , it is useful during data integration.

+

Features:

+
    +
  • Execution of a Talend job by importing its archive .
    • +
  • Visualization of the job's execution logs (creation date, end date ,details ,state ,execution duration).
    • +
  • Offers the possibility to add context variables to manage various execution types (database to use ,username ,password ,port ,host...).
    • +
  • Definition of children jobs.
    • +
  • Definition of the parent job.
    • +
  • Passing a context environment to children jobs.
    • +
  • Control the allocated memory for the job execution (by specifying the argument in the args field).
    • +
  • Jobs and their executions logs storage .
    • +
+

Table of contents

+ +
+

Usage

+

To run a Talend job:

+
    +
  1. Go to Settings > Talend Jobs menu.
    2. +
  2. Press the button Create.
    3. +
  3. Insert the name of the job (the name should be the same name used for the job in Talend studio ).
    4. +
  4. Upload the job's archive file (zip format) to archive field .
    5. +
  5. Click on Run button.
    6. +
  6. Click on Refresh logs button.
    7. +
+
+Job creation +
+

To propagate a context environment to children jobs :

+
    +
  1. Add the concerned children jobs to Children field .
    2. +
  2. Click on Propagate context button.
    3. +
+
+Propagate Context +
+

Parameters you can specify :

+
    +
  1. Path : to specify the path to the job's directory .
    2. +
  2. Args : add -Xms64M or -Xmx1024M to control the available memory for the job's execution.
    3. +
  3. Loop : to specify the number of times the job will be executed.
    4. +
  4. Parent : to indicate the parent job.
    5. +
  5. Children : to add children jobs.
    6. +
  6. Context : to specify job's context parameters.
    7. +
+
+Path Update +
+
+Allocated memory control +
+
+Context Update +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +here.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

GDPR / EU Privacy

+

This addons does not collect any data and does not set any browser cookies.

+
+
+

Credits

+
+

Contributors

+
    +
  • Corentin POUHET-BRUNERIE
    • +
+
+
+

Maintainer

+

This module is maintained by Smile SA.

+

Since 1991 Smile has been a pioneer of technology and also the European expert in open source solutions.

+Smile SA +

This module is part of the odoo-addons project on GitHub.

+

You are welcome to contribute.

+
+
+
+ + diff --git a/smile_talend_job/static/description/job_creation.png b/smile_talend_job/static/description/job_creation.png new file mode 100644 index 000000000..98b3ab8c8 Binary files /dev/null and b/smile_talend_job/static/description/job_creation.png differ diff --git a/smile_talend_job/static/description/path.png b/smile_talend_job/static/description/path.png new file mode 100644 index 000000000..5126d79ec Binary files /dev/null and b/smile_talend_job/static/description/path.png differ diff --git a/smile_talend_job/tests/__init__.py b/smile_talend_job/tests/__init__.py new file mode 100644 index 000000000..92f832cea --- /dev/null +++ b/smile_talend_job/tests/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import test_talend_job diff --git a/smile_talend_job/tests/test_talend_job.py b/smile_talend_job/tests/test_talend_job.py new file mode 100644 index 000000000..955db36d9 --- /dev/null +++ b/smile_talend_job/tests/test_talend_job.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo import registry +from odoo.tests.common import TransactionCase + + +class TestTalendJob(TransactionCase): + + def setUp(self): + super(TestTalendJob, self).setUp() + self.main_job = self.env.ref('smile_talend_job.main_job') + self.main_job.child_ids.unlink() + + def test_010_talend_job_single(self): + logs_nb = len(self.main_job.log_ids) + self.main_job.with_context(in_new_thread=False).run() + with registry(self.env.cr.dbname).cursor() as new_cr: + self.assertEquals(len(self.main_job.with_env( + self.env(cr=new_cr)).log_ids), logs_nb + 1) + + def test_020_talend_job_multiple(self): + second_job = self.main_job.copy( + default={'parent_id': self.main_job.id}) + second_job._cr.commit() + self.assertEquals(len(second_job.log_ids), 0) + self.main_job.with_context(in_new_thread=False).run_only() + with registry(self.env.cr.dbname).cursor() as new_cr: + self.assertEquals(len(second_job.with_env( + self.env(cr=new_cr)).log_ids), 0) + self.main_job.with_context(in_new_thread=False).run() + with registry(self.env.cr.dbname).cursor() as new_cr: + self.assertEquals(len(second_job.with_env( + self.env(cr=new_cr)).log_ids), 1) + + def test_030_talend_job_context_propagation(self): + second_job = self.main_job.copy( + default={'parent_id': self.main_job.id}) + self.main_job.context = "Main context" + self.assertFalse(second_job.context) + self.main_job.propagate_context() + self.assertEquals(second_job.context, self.main_job.context) diff --git a/smile_talend_job/tools/__init__.py b/smile_talend_job/tools/__init__.py new file mode 100644 index 000000000..3f483812c --- /dev/null +++ b/smile_talend_job/tools/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from .misc import * diff --git a/smile_talend_job/tools/misc.py b/smile_talend_job/tools/misc.py new file mode 100644 index 000000000..f00c51573 --- /dev/null +++ b/smile_talend_job/tools/misc.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# (C) 2020 Smile () +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +def s2human(time): + """ + Copy from https://github.com/odoo/odoo-extra/blob/master/runbot/runbot.py + """ + for delay, desc in [(86400, 'd'), (3600, 'h'), (60, 'm')]: + if time >= delay: + return str(int(time / delay)) + desc + return str(int(time)) + "s" diff --git a/smile_talend_job/views/talend_job_logs_view.xml b/smile_talend_job/views/talend_job_logs_view.xml new file mode 100644 index 000000000..a130020af --- /dev/null +++ b/smile_talend_job/views/talend_job_logs_view.xml @@ -0,0 +1,36 @@ + + + + + + talend.job.logs.form + talend.job.logs + +
+
+
+ +
+

+ +

+ + + + + +
+ + + + + + +
+ + + + + diff --git a/smile_talend_job/views/talend_job_view.xml b/smile_talend_job/views/talend_job_view.xml new file mode 100644 index 000000000..2259f59c8 --- /dev/null +++ b/smile_talend_job/views/talend_job_view.xml @@ -0,0 +1,98 @@ + + + + + + talend.job.form + talend.job + +
+
+
+ +
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + +