Tripal v1.0 (6.x-1.0)
|
Go to the source code of this file.
Functions | |
tripal_bulk_loader_init () | |
tripal_bulk_loader_menu () | |
tripal_bulk_loader_theme () | |
tripal_bulk_loader_access ($op, $node, $account) | |
tripal_bulk_loader_perm () | |
tripal_bulk_loader_node_info () | |
tripal_bulk_loader_form ($node, $form_state) | |
tripal_bulk_loader_load ($node) | |
tripal_bulk_loader_insert ($node) | |
tripal_bulk_loader_delete ($node) | |
tripal_bulk_loader_update ($node) | |
tripal_bulk_loader_preprocess_tripal_bulk_loader_template (&$variables) | |
tripal_bulk_loader_progess_file_get_progress ($job_id, $update_progress=TRUE) | |
tripal_bulk_loader_job_describe_args ($callback, $args) | |
tripal_bulk_loader_coder_ignore () |
The functions for the Tripal bulk loader.
Definition in file tripal_bulk_loader.module.
tripal_bulk_loader_access | ( | $ | op, |
$ | node, | ||
$ | account | ||
) |
Implements hook_access
Definition at line 194 of file tripal_bulk_loader.module.
{ if ($op == 'create') { if (!user_access('create tripal_bulk_loader', $account)) { return FALSE; } } if ($op == 'update') { if (!user_access('edit tripal_bulk_loader', $account)) { return FALSE; } } if ($op == 'delete') { if (!user_access('delete tripal_bulk_loader', $account)) { return FALSE; } } if ($op == 'view') { if (!user_access('access tripal_bulk_loader', $account)) { return FALSE; } } return NULL; }
tripal_bulk_loader_coder_ignore | ( | ) |
Implements hook_coder_ignore(). Defines the path to the file (tripal_bulk_loader.coder_ignores.txt) where ignore rules for coder are stored
Definition at line 576 of file tripal_bulk_loader.module.
{ return array( 'path' => drupal_get_path('module', 'tripal_bulk_loader'), 'line prefix' => drupal_get_path('module', 'tripal_bulk_loader'), ); }
tripal_bulk_loader_delete | ( | $ | node | ) |
Implements node_delete Deletes the data when the delete button on the node form is clicked
Definition at line 428 of file tripal_bulk_loader.module.
{
$sql = "DELETE FROM {tripal_bulk_loader} WHERE nid = %d";
db_query($sql, $node->nid);
}
tripal_bulk_loader_form | ( | $ | node, |
$ | form_state | ||
) |
Implements node_form Used to gather the extra details stored with a Bulk Loading Job Node
Definition at line 254 of file tripal_bulk_loader.module.
{ $form = array(); if (isset($form_state['values'])) { $node = $form_state['values'] + (array)$node; $node = (object) $node; } $sql = "SELECT * FROM {tripal_bulk_loader_template}"; $results = db_query($sql); $templates = array(); while ($template = db_fetch_object ($results)) { $templates [$template->template_id] = $template->name; } if (!$templates) { $form['label'] = array( '#type' => 'item', '#description' => t("Loader template needs to be created before any bulk loader can be added. Go to 'Tripal Management > Bulk Loader Template' to create the template."), '#weight' => -10, ); return $form; } $form['loader'] = array( '#type' => 'fieldset', '#title' => t('Basic Details'), ); $form['loader']['loader_name'] = array( '#type' => 'textfield', '#title' => t('Loading Job Name'), '#weight' => -10, '#required' => TRUE, '#default_value' => $node->loader_name ); $form['loader']['template_id'] = array( '#type' => 'select', '#title' => t('Template'), '#description' => t('Please specify a template for this loader'), '#options' => $templates, '#weight' => -9, '#required' => TRUE, '#default_value' => $node->template_id, ); $form['loader']['file']= array( '#type' => 'textfield', '#title' => t('Data File'), '#description' => t('Please specify the data file to be loaded. This must be a tab-delimited text file with UNIX line endings.'), '#weight' => -8, '#default_value' => $node->file, '#maxlength' => 1024, ); $form['loader']['has_header'] = array( '#type' => 'radios', '#title' => t('File has a Header'), '#options' => array( 1 => 'Yes', 2 => 'No'), '#weight' => -7, '#default_value' => $node->file_has_header, ); $form['loader']['keep_track_inserted'] = array( '#type' => 'radios', '#title' => t('Keep track of inserted record IDs'), '#description' => t('This enables the ability to revert an entire loading job even if ' .'it completed successfully. Furthermore, it displays the number of records ' .'successfully inserted into each table.'), '#options' => array( 1 => 'Yes', 0 => 'No'), '#weight' => -7, '#default_value' => (isset($node->keep_track_inserted)) ? $node->keep_track_inserted : variable_get('tripal_bulk_loader_keep_track_inserted', FALSE), ); return $form; }
tripal_bulk_loader_init | ( | ) |
Implements hook_init Used to add stylesheets and javascript files to the header
Definition at line 21 of file tripal_bulk_loader.module.
{ // Add javascript and style sheet drupal_add_css(drupal_get_path('theme', 'tripal') . '/css/tripal_bulk_loader.css'); drupal_add_js(drupal_get_path('theme', 'tripal') . '/js/tripal_bulk_loader.js'); }
tripal_bulk_loader_insert | ( | $ | node | ) |
Implements node_insert Insert the data from the node form on Create content
Definition at line 409 of file tripal_bulk_loader.module.
{ // Insert into tripal_bulk_loader $sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file, file_has_header, job_status, keep_track_inserted) VALUES (%d, '%s', %d, '%s', %d, '%s', %d)"; db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, 'Initialized', $node->keep_track_inserted); // Update title $node->title =$node->loader_name; drupal_write_record('node', $node, 'nid'); drupal_write_record('node_revision', $node, 'nid'); drupal_set_message(t('After reviewing the details, please Submit this Job (by clicking the "Submit Job" button below). No data will be loaded until the submitted job is reached in the queue.')); }
tripal_bulk_loader_job_describe_args | ( | $ | callback, |
$ | args | ||
) |
Implements hook_job_describe_args() Specifically to make viewing past tripal jobs more readable for jobs registered by this module
$callback The callback passed into tripal_add_job()
$args | The arguements passed into tripal_add_job() |
Definition at line 560 of file tripal_bulk_loader.module.
tripal_bulk_loader_load | ( | $ | node | ) |
Implements node_load
Definition at line 337 of file tripal_bulk_loader.module.
{ $sql = "SELECT * FROM {tripal_bulk_loader} WHERE nid = %d"; $node = db_fetch_object(db_query($sql, $node->nid)); $node->title = 'Bulk Loading Job: ' . $node->loader_name; // Add job details $progress = tripal_bulk_loader_progess_file_get_progress($node->job_id); $sql = "SELECT * FROM {tripal_jobs} WHERE job_id=%d"; $node->job = db_fetch_object(db_query($sql, $node->job_id)); // Add the loader template $sql = "SELECT * FROM {tripal_bulk_loader_template} WHERE template_id=%d"; $results = db_fetch_object(db_query($sql, $node->template_id)); $template = unserialize($results->template_array); $node->template = $results; $node->template->template_array = $template; // Add inserted records $sql = 'SELECT * FROM {tripal_bulk_loader_inserted} WHERE nid=%d'; $resource = db_query($sql, $node->nid); while ($r = db_fetch_object($resource)) { $r->num_inserted = sizeof(preg_split('/,/', $r->ids_inserted)); $node->inserted_records->{$r->table_inserted_into} = $r; } // Add exposed field list $node->exposed_fields = array(); if ($template) { foreach ($template as $record_id => $record) { foreach ($record['fields'] as $field_id => $field) { if ($field['exposed']) { $node->exposed_fields[] = array( 'record_id' => $record_id, 'field_id' => $field_id, 'title' => $field['title'], ); } } } if (empty($node->exposed_fields)) { $node->exposed_fields[] = array(); } } // Add constants $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY group_id, record_id, field_id'; $resource = db_query($sql, $node->nid); while ($r = db_fetch_object($resource)) { $node->constants[$r->group_id][$r->record_id][$r->field_id] = array( 'constant_id' => $r->constant_id, 'group_id' => $r->group_id, 'chado_table' => $r->chado_table, 'chado_field' => $r->chado_field, 'record_id' => $r->record_id, 'field_id' => $r->field_id, 'value' => $r->value ); } if (!$node->constants) { $node->constants[] = array(); } return $node; }
tripal_bulk_loader_menu | ( | ) |
Implements hook_menu
Definition at line 30 of file tripal_bulk_loader.module.
{ $items = array(); // Bulk Loading Job Node $items['node/%node/constants/%/edit'] = array( 'title' => 'Edit Constant Set', 'description' => 'Edit a group of constants associated with the current bulk loader', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_edit_constant_set_form', 1, 3), 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); $items['node/%node/constants/%/delete'] = array( 'title' => 'Delete Constant Set', 'description' => 'Delete a group of constants associated with the current bulk loader', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_delete_constant_set_form', 1, 3), 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); // Admin pages ----------------- $items['admin/tripal/tripal_bulk_loader_template'] = array( 'title' => 'Bulk Loader', 'description' => 'Templates for loading tab-delimited data', 'page callback' => 'tripal_bulk_loader_admin_template', 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/configure'] = array( 'title' => 'Configure', 'description' => 'Configuration of global options related to bulk loading jobs', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_configuration_form'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/manage_templates'] = array( 'title' => 'Manage Templates', 'description' => 'Create/Update/Delete/Import/Export Templates', 'page callback' => 'tripal_bulk_loader_admin_manage_templates', 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/jobs'] = array( 'title' => 'Bulk Loader Jobs', 'description' => 'Listing of Bulk Loading Jobs', 'page callback' => 'tripal_bulk_loader_admin_jobs', 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); // Create/Edit Template -------- $items['admin/tripal/tripal_bulk_loader_template/manage_templates/create'] = array( 'title' => 'Create Template', 'description' => 'Create loader template for loading tab-delimited data', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'create'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/manage_templates/edit'] = array( 'title' => 'Edit Template', 'description' => 'Edit loader template for loading tab-delimited data', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/edit_record'] = array( 'title' => 'Edit Template Record', 'description' => 'Edit a record in an existing tripal bulk loader template.', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_edit_template_record_form'), 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); $items['admin/tripal/tripal_bulk_loader_template/add_field'] = array( 'title' => 'Add Template Field', 'description' => 'Add a template field to an existing tripal bulk loader template.', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_add_template_field_form'), 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); $items['admin/tripal/tripal_bulk_loader_template/edit_field'] = array( 'title' => 'Edit Template Field', 'description' => 'Edit an existing field from a tripal bulk loader template.', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_edit_template_field_form'), 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); // Delete Template ----- $items['admin/tripal/tripal_bulk_loader_template/manage_templates/delete'] = array( 'title' => 'Delete Template', 'description' => 'Delete bulk loader template', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_delete_template_base_form'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); // Import/Export --------- $items['admin/tripal/tripal_bulk_loader_template/manage_templates/import'] = array( 'title' => 'Import Template', 'description' => 'Import Loaders', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_import_export_template_form', 'import'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); $items['admin/tripal/tripal_bulk_loader_template/manage_templates/export'] = array( 'title' => 'Export Template', 'description' => 'Export Loaders', 'page callback' => 'drupal_get_form', 'page arguments' => array('tripal_bulk_loader_import_export_template_form', 'export'), 'access arguments' => array('administer site configuration'), 'type' => MENU_NORMAL_ITEM, ); // AHAH --------- $items['admin/tripal/tripal_bulk_loader_template/add_field_ahah'] = array( 'page callback' => 'tripal_bulk_loader_add_field_ahah', 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); $items['admin/tripal/tripal_bulk_loader_template/edit_field_ahah'] = array( 'page callback' => 'tripal_bulk_loader_edit_field_ahah', 'access arguments' => array('administer site configuration'), 'type' => MENU_CALLBACK, ); return $items; }
tripal_bulk_loader_node_info | ( | ) |
Implements hook_node_info
Definition at line 237 of file tripal_bulk_loader.module.
{ $nodes = array(); $nodes['tripal_bulk_loader'] = array( 'name' => t('Bulk Loading Job'), 'module' => 'tripal_bulk_loader', 'description' => t('A bulk loader for inserting tab-delimited data into chado database'), 'has_title' => TRUE, 'has_body' => FALSE, 'locked' => TRUE ); return $nodes; }
tripal_bulk_loader_perm | ( | ) |
Implements hook_perm
Definition at line 221 of file tripal_bulk_loader.module.
{ return array( 'access tripal_bulk_loader', 'create tripal_bulk_loader', 'delete tripal_bulk_loader', 'edit tripal_bulk_loader', ); }
tripal_bulk_loader_preprocess_tripal_bulk_loader_template | ( | &$ | variables | ) |
Preprocessor function for the tripal_bulk_loader template
Definition at line 465 of file tripal_bulk_loader.module.
{ $sql = "SELECT * FROM {tripal_bulk_loader_template} WHERE template_id=%d"; $template = db_fetch_object(db_query($sql, $variables['template_id'])); $template->template_array = unserialize($template->template_array); $variables['template'] = $template; }
tripal_bulk_loader_progess_file_get_progress | ( | $ | job_id, |
$ | update_progress = TRUE |
||
) |
Get the progress of the current constant set from the progress file
When transactions are used, database updates to drupal cannot be made. Thus a separate method to keep track of progress was implemented: save a period to the file for each record successfully inserted; each line in the file represents a processed line.
$job_id | The id of the tripal job to check the progress of |
$node | The tripal_bulk_loader node associated with the job |
Definition at line 493 of file tripal_bulk_loader.module.
{ $filename = '/tmp/tripal_bulk_loader_progress-' . $job_id . '.out'; if (!file_exists($filename)) { return (object) array(); } $num_lines = trim(`wc --lines < $filename`); $num_records = trim(`grep -o "." $filename | wc --lines`); $job = db_fetch_object(db_query("SELECT j.*, b.file, b.file_has_header, c.num as num_constant_sets FROM {tripal_jobs} j LEFT JOIN {tripal_bulk_loader} b ON b.job_id=j.job_id LEFT JOIN ( SELECT nid, count(distinct(group_id)) as num FROM {tripal_bulk_loader_constants} GROUP BY nid ) c ON c.nid=b.nid WHERE j.job_id=%d", $job_id)); if ($job->num_constant_sets) { $num_constant_sets_loaded = round($job->progress / (100 / $job->num_constant_sets), 4); // If the next constant set has started loading if ($job->num_constant_sets != $num_constant_sets_loaded) { // total lines in input file $total_lines = trim(`wc --lines < $job->file`); if ($job->file_has_header) { $total_lines--; } // percent of the current constant set loaded $percent = round($num_lines/$total_lines * 100, 2); // percent of the total job = (<# fully loaded constant sets> * 100 ) // + <percent of current constant set> // / <total number of constant sets> $total_percent = (($num_constant_sets_loaded * 100) + $percent) / $job->num_constant_sets; // update the progress of the job if ($update_progress AND ($percent != 0 OR $percent != 100)) { tripal_job_set_progress($job_id, round($total_percent, 0)); } } } return (object) array( 'num_lines' => $num_lines, 'total_lines' => $total_lines, 'percent_file' => $percent, 'num_constant_sets_loaded' => $num_constant_sets_loaded, 'total_percent' => $total_percent, 'num_records' => $num_records ); }
tripal_bulk_loader_theme | ( | ) |
Implements hook_theme
Definition at line 167 of file tripal_bulk_loader.module.
{ return array( 'tripal_bulk_loader_set_constants_form' => array( 'arguments' => array('form' => NULL), ), 'tripal_bulk_loader_template' => array( 'arguments' => array('template_id' => NULL), 'template' => 'tripal_bulk_loader_template' ), 'tripal_bulk_loader_modify_template_base_form' => array( 'arguments' => array('form' => NULL), 'template' => 'tripal_bulk_loader_modify_template_base_form', ), 'tripal_bulk_loader_edit_template_field_form' => array( 'arguments' => array('form' => NULL), 'template' => 'tripal_bulk_loader_edit_template_field_form', ), 'tripal_bulk_loader_add_template_field_form' => array( 'arguments' => array('form' => NULL), 'template' => 'tripal_bulk_loader_add_template_field_form', ), ); }
tripal_bulk_loader_update | ( | $ | node | ) |
Implements node_update Updates the data submitted by the node form on edit
Definition at line 437 of file tripal_bulk_loader.module.
{ // Update tripal_bulk_loader $sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s', file_has_header = '%s', keep_track_inserted = %d WHERE nid = %d"; db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, $node->keep_track_inserted, $node->nid); // Add a job if the user want to load the data global $user; if ($node->job) { $job_args[0] =$node->loader_name; $job_args[1] = $node->template_id; $job_args[2] = $node->file; if (is_readable($node->file)) { $fname = preg_replace("/.*\/(.*)/", "$1", $node->file); tripal_add_job("Bulk Load: $fname", 'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid); } else { drupal_set_message(t("Can not open %file. Job not scheduled.", array('%file' => $node->file))); } } }