Tripal v1.0 (6.x-1.0)
|
00001 <?php 00009 include('tripal_bulk_loader.loader.inc'); 00010 include('tripal_bulk_loader.constants.inc'); 00011 include('tripal_bulk_loader.admin.inc'); 00012 include('tripal_bulk_loader.admin.templates.inc'); 00013 00014 // API 00015 include('api/tripal_bulk_loader.api.templates.inc'); 00016 00021 function tripal_bulk_loader_init() { 00022 // Add javascript and style sheet 00023 drupal_add_css(drupal_get_path('theme', 'tripal') . '/css/tripal_bulk_loader.css'); 00024 drupal_add_js(drupal_get_path('theme', 'tripal') . '/js/tripal_bulk_loader.js'); 00025 } 00026 00030 function tripal_bulk_loader_menu() { 00031 $items = array(); 00032 // Bulk Loading Job Node 00033 $items['node/%node/constants/%/edit'] = array( 00034 'title' => 'Edit Constant Set', 00035 'description' => 'Edit a group of constants associated with the current bulk loader', 00036 'page callback' => 'drupal_get_form', 00037 'page arguments' => array('tripal_bulk_loader_edit_constant_set_form', 1, 3), 00038 'access arguments' => array('administer site configuration'), 00039 'type' => MENU_CALLBACK, 00040 ); 00041 $items['node/%node/constants/%/delete'] = array( 00042 'title' => 'Delete Constant Set', 00043 'description' => 'Delete a group of constants associated with the current bulk loader', 00044 'page callback' => 'drupal_get_form', 00045 'page arguments' => array('tripal_bulk_loader_delete_constant_set_form', 1, 3), 00046 'access arguments' => array('administer site configuration'), 00047 'type' => MENU_CALLBACK, 00048 ); 00049 00050 // Admin pages ----------------- 00051 $items['admin/tripal/tripal_bulk_loader_template'] = array( 00052 'title' => 'Bulk Loader', 00053 'description' => 'Templates for loading tab-delimited data', 00054 'page callback' => 'tripal_bulk_loader_admin_template', 00055 'access arguments' => array('administer site configuration'), 00056 'type' => MENU_NORMAL_ITEM, 00057 ); 00058 $items['admin/tripal/tripal_bulk_loader_template/configure'] = array( 00059 'title' => 'Configure', 00060 'description' => 'Configuration of global options related to bulk loading jobs', 00061 'page callback' => 'drupal_get_form', 00062 'page arguments' => array('tripal_bulk_loader_configuration_form'), 00063 'access arguments' => array('administer site configuration'), 00064 'type' => MENU_NORMAL_ITEM, 00065 ); 00066 $items['admin/tripal/tripal_bulk_loader_template/manage_templates'] = array( 00067 'title' => 'Manage Templates', 00068 'description' => 'Create/Update/Delete/Import/Export Templates', 00069 'page callback' => 'tripal_bulk_loader_admin_manage_templates', 00070 'access arguments' => array('administer site configuration'), 00071 'type' => MENU_NORMAL_ITEM, 00072 ); 00073 $items['admin/tripal/tripal_bulk_loader_template/jobs'] = array( 00074 'title' => 'Bulk Loader Jobs', 00075 'description' => 'Listing of Bulk Loading Jobs', 00076 'page callback' => 'tripal_bulk_loader_admin_jobs', 00077 'access arguments' => array('administer site configuration'), 00078 'type' => MENU_NORMAL_ITEM, 00079 ); 00080 00081 // Create/Edit Template -------- 00082 $items['admin/tripal/tripal_bulk_loader_template/manage_templates/create'] = array( 00083 'title' => 'Create Template', 00084 'description' => 'Create loader template for loading tab-delimited data', 00085 'page callback' => 'drupal_get_form', 00086 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'create'), 00087 'access arguments' => array('administer site configuration'), 00088 'type' => MENU_NORMAL_ITEM, 00089 ); 00090 $items['admin/tripal/tripal_bulk_loader_template/manage_templates/edit'] = array( 00091 'title' => 'Edit Template', 00092 'description' => 'Edit loader template for loading tab-delimited data', 00093 'page callback' => 'drupal_get_form', 00094 'page arguments' => array('tripal_bulk_loader_modify_template_base_form', 'edit'), 00095 'access arguments' => array('administer site configuration'), 00096 'type' => MENU_NORMAL_ITEM, 00097 ); 00098 $items['admin/tripal/tripal_bulk_loader_template/edit_record'] = array( 00099 'title' => 'Edit Template Record', 00100 'description' => 'Edit a record in an existing tripal bulk loader template.', 00101 'page callback' => 'drupal_get_form', 00102 'page arguments' => array('tripal_bulk_loader_edit_template_record_form'), 00103 'access arguments' => array('administer site configuration'), 00104 'type' => MENU_CALLBACK, 00105 ); 00106 $items['admin/tripal/tripal_bulk_loader_template/add_field'] = array( 00107 'title' => 'Add Template Field', 00108 'description' => 'Add a template field to an existing tripal bulk loader template.', 00109 'page callback' => 'drupal_get_form', 00110 'page arguments' => array('tripal_bulk_loader_add_template_field_form'), 00111 'access arguments' => array('administer site configuration'), 00112 'type' => MENU_CALLBACK, 00113 ); 00114 $items['admin/tripal/tripal_bulk_loader_template/edit_field'] = array( 00115 'title' => 'Edit Template Field', 00116 'description' => 'Edit an existing field from a tripal bulk loader template.', 00117 'page callback' => 'drupal_get_form', 00118 'page arguments' => array('tripal_bulk_loader_edit_template_field_form'), 00119 'access arguments' => array('administer site configuration'), 00120 'type' => MENU_CALLBACK, 00121 ); 00122 // Delete Template ----- 00123 $items['admin/tripal/tripal_bulk_loader_template/manage_templates/delete'] = array( 00124 'title' => 'Delete Template', 00125 'description' => 'Delete bulk loader template', 00126 'page callback' => 'drupal_get_form', 00127 'page arguments' => array('tripal_bulk_loader_delete_template_base_form'), 00128 'access arguments' => array('administer site configuration'), 00129 'type' => MENU_NORMAL_ITEM, 00130 ); 00131 // Import/Export --------- 00132 $items['admin/tripal/tripal_bulk_loader_template/manage_templates/import'] = array( 00133 'title' => 'Import Template', 00134 'description' => 'Import Loaders', 00135 'page callback' => 'drupal_get_form', 00136 'page arguments' => array('tripal_bulk_loader_import_export_template_form', 'import'), 00137 'access arguments' => array('administer site configuration'), 00138 'type' => MENU_NORMAL_ITEM, 00139 ); 00140 $items['admin/tripal/tripal_bulk_loader_template/manage_templates/export'] = array( 00141 'title' => 'Export Template', 00142 'description' => 'Export Loaders', 00143 'page callback' => 'drupal_get_form', 00144 'page arguments' => array('tripal_bulk_loader_import_export_template_form', 'export'), 00145 'access arguments' => array('administer site configuration'), 00146 'type' => MENU_NORMAL_ITEM, 00147 ); 00148 // AHAH --------- 00149 $items['admin/tripal/tripal_bulk_loader_template/add_field_ahah'] = array( 00150 'page callback' => 'tripal_bulk_loader_add_field_ahah', 00151 'access arguments' => array('administer site configuration'), 00152 'type' => MENU_CALLBACK, 00153 ); 00154 $items['admin/tripal/tripal_bulk_loader_template/edit_field_ahah'] = array( 00155 'page callback' => 'tripal_bulk_loader_edit_field_ahah', 00156 'access arguments' => array('administer site configuration'), 00157 'type' => MENU_CALLBACK, 00158 ); 00159 00160 return $items; 00161 } 00162 00163 00167 function tripal_bulk_loader_theme() { 00168 return array( 00169 'tripal_bulk_loader_set_constants_form' => array( 00170 'arguments' => array('form' => NULL), 00171 ), 00172 'tripal_bulk_loader_template' => array( 00173 'arguments' => array('template_id' => NULL), 00174 'template' => 'tripal_bulk_loader_template' 00175 ), 00176 'tripal_bulk_loader_modify_template_base_form' => array( 00177 'arguments' => array('form' => NULL), 00178 'template' => 'tripal_bulk_loader_modify_template_base_form', 00179 ), 00180 'tripal_bulk_loader_edit_template_field_form' => array( 00181 'arguments' => array('form' => NULL), 00182 'template' => 'tripal_bulk_loader_edit_template_field_form', 00183 ), 00184 'tripal_bulk_loader_add_template_field_form' => array( 00185 'arguments' => array('form' => NULL), 00186 'template' => 'tripal_bulk_loader_add_template_field_form', 00187 ), 00188 ); 00189 } 00190 00194 function tripal_bulk_loader_access($op, $node, $account) { 00195 if ($op == 'create') { 00196 if (!user_access('create tripal_bulk_loader', $account)) { 00197 return FALSE; 00198 } 00199 } 00200 if ($op == 'update') { 00201 if (!user_access('edit tripal_bulk_loader', $account)) { 00202 return FALSE; 00203 } 00204 } 00205 if ($op == 'delete') { 00206 if (!user_access('delete tripal_bulk_loader', $account)) { 00207 return FALSE; 00208 } 00209 } 00210 if ($op == 'view') { 00211 if (!user_access('access tripal_bulk_loader', $account)) { 00212 return FALSE; 00213 } 00214 } 00215 return NULL; 00216 } 00217 00221 function tripal_bulk_loader_perm() { 00222 return array( 00223 'access tripal_bulk_loader', 00224 'create tripal_bulk_loader', 00225 'delete tripal_bulk_loader', 00226 'edit tripal_bulk_loader', 00227 ); 00228 } 00229 00231 // Node Functions 00233 00237 function tripal_bulk_loader_node_info() { 00238 $nodes = array(); 00239 $nodes['tripal_bulk_loader'] = array( 00240 'name' => t('Bulk Loading Job'), 00241 'module' => 'tripal_bulk_loader', 00242 'description' => t('A bulk loader for inserting tab-delimited data into chado database'), 00243 'has_title' => TRUE, 00244 'has_body' => FALSE, 00245 'locked' => TRUE 00246 ); 00247 return $nodes; 00248 } 00249 00254 function tripal_bulk_loader_form($node, $form_state) { 00255 $form = array(); 00256 00257 if (isset($form_state['values'])) { 00258 $node = $form_state['values'] + (array)$node; 00259 $node = (object) $node; 00260 } 00261 00262 $sql = "SELECT * FROM {tripal_bulk_loader_template}"; 00263 $results = db_query($sql); 00264 $templates = array(); 00265 while ($template = db_fetch_object ($results)) { 00266 $templates [$template->template_id] = $template->name; 00267 } 00268 00269 if (!$templates) { 00270 $form['label'] = array( 00271 '#type' => 'item', 00272 '#description' => t("Loader template needs to be created before any bulk loader can be added. Go to 'Tripal Management > Bulk Loader Template' to create the template."), 00273 '#weight' => -10, 00274 ); 00275 00276 return $form; 00277 } 00278 00279 $form['loader'] = array( 00280 '#type' => 'fieldset', 00281 '#title' => t('Basic Details'), 00282 ); 00283 00284 $form['loader']['loader_name'] = array( 00285 '#type' => 'textfield', 00286 '#title' => t('Loading Job Name'), 00287 '#weight' => -10, 00288 '#required' => TRUE, 00289 '#default_value' => $node->loader_name 00290 ); 00291 00292 $form['loader']['template_id'] = array( 00293 '#type' => 'select', 00294 '#title' => t('Template'), 00295 '#description' => t('Please specify a template for this loader'), 00296 '#options' => $templates, 00297 '#weight' => -9, 00298 '#required' => TRUE, 00299 '#default_value' => $node->template_id, 00300 ); 00301 00302 $form['loader']['file']= array( 00303 '#type' => 'textfield', 00304 '#title' => t('Data File'), 00305 '#description' => t('Please specify the data file to be loaded. This must be a tab-delimited text file with UNIX line endings.'), 00306 '#weight' => -8, 00307 '#default_value' => $node->file, 00308 '#maxlength' => 1024, 00309 ); 00310 00311 $form['loader']['has_header'] = array( 00312 '#type' => 'radios', 00313 '#title' => t('File has a Header'), 00314 '#options' => array( 1 => 'Yes', 2 => 'No'), 00315 '#weight' => -7, 00316 '#default_value' => $node->file_has_header, 00317 ); 00318 00319 $form['loader']['keep_track_inserted'] = array( 00320 '#type' => 'radios', 00321 '#title' => t('Keep track of inserted record IDs'), 00322 '#description' => t('This enables the ability to revert an entire loading job even if ' 00323 .'it completed successfully. Furthermore, it displays the number of records ' 00324 .'successfully inserted into each table.'), 00325 '#options' => array( 1 => 'Yes', 0 => 'No'), 00326 '#weight' => -7, 00327 '#default_value' => (isset($node->keep_track_inserted)) ? $node->keep_track_inserted : variable_get('tripal_bulk_loader_keep_track_inserted', FALSE), 00328 ); 00329 00330 return $form; 00331 } 00332 00333 00337 function tripal_bulk_loader_load($node) { 00338 $sql = "SELECT * FROM {tripal_bulk_loader} WHERE nid = %d"; 00339 $node = db_fetch_object(db_query($sql, $node->nid)); 00340 00341 $node->title = 'Bulk Loading Job: ' . $node->loader_name; 00342 00343 // Add job details 00344 $progress = tripal_bulk_loader_progess_file_get_progress($node->job_id); 00345 $sql = "SELECT * FROM {tripal_jobs} WHERE job_id=%d"; 00346 $node->job = db_fetch_object(db_query($sql, $node->job_id)); 00347 00348 // Add the loader template 00349 $sql = "SELECT * FROM {tripal_bulk_loader_template} WHERE template_id=%d"; 00350 $results = db_fetch_object(db_query($sql, $node->template_id)); 00351 $template = unserialize($results->template_array); 00352 $node->template = $results; 00353 $node->template->template_array = $template; 00354 00355 // Add inserted records 00356 $sql = 'SELECT * FROM {tripal_bulk_loader_inserted} WHERE nid=%d'; 00357 $resource = db_query($sql, $node->nid); 00358 while ($r = db_fetch_object($resource)) { 00359 $r->num_inserted = sizeof(preg_split('/,/', $r->ids_inserted)); 00360 $node->inserted_records->{$r->table_inserted_into} = $r; 00361 } 00362 00363 // Add exposed field list 00364 $node->exposed_fields = array(); 00365 if ($template) { 00366 foreach ($template as $record_id => $record) { 00367 foreach ($record['fields'] as $field_id => $field) { 00368 if ($field['exposed']) { 00369 $node->exposed_fields[] = array( 00370 'record_id' => $record_id, 00371 'field_id' => $field_id, 00372 'title' => $field['title'], 00373 ); 00374 } 00375 } 00376 } 00377 00378 if (empty($node->exposed_fields)) { 00379 $node->exposed_fields[] = array(); 00380 } 00381 } 00382 00383 00384 // Add constants 00385 $sql = 'SELECT * FROM {tripal_bulk_loader_constants} WHERE nid=%d ORDER BY group_id, record_id, field_id'; 00386 $resource = db_query($sql, $node->nid); 00387 while ($r = db_fetch_object($resource)) { 00388 $node->constants[$r->group_id][$r->record_id][$r->field_id] = array( 00389 'constant_id' => $r->constant_id, 00390 'group_id' => $r->group_id, 00391 'chado_table' => $r->chado_table, 00392 'chado_field' => $r->chado_field, 00393 'record_id' => $r->record_id, 00394 'field_id' => $r->field_id, 00395 'value' => $r->value 00396 ); 00397 } 00398 if (!$node->constants) { 00399 $node->constants[] = array(); 00400 } 00401 00402 return $node; 00403 } 00404 00409 function tripal_bulk_loader_insert($node) { 00410 00411 // Insert into tripal_bulk_loader 00412 $sql = "INSERT INTO {tripal_bulk_loader} (nid, loader_name, template_id, file, file_has_header, job_status, keep_track_inserted) VALUES (%d, '%s', %d, '%s', %d, '%s', %d)"; 00413 db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, 'Initialized', $node->keep_track_inserted); 00414 00415 // Update title 00416 $node->title =$node->loader_name; 00417 drupal_write_record('node', $node, 'nid'); 00418 drupal_write_record('node_revision', $node, 'nid'); 00419 00420 drupal_set_message(t('After reviewing the details, please Submit this Job (by clicking the "Submit Job" button below). No data will be loaded until the submitted job is reached in the queue.')); 00421 00422 } 00423 00428 function tripal_bulk_loader_delete($node) { 00429 $sql = "DELETE FROM {tripal_bulk_loader} WHERE nid = %d"; 00430 db_query($sql, $node->nid); 00431 } 00432 00437 function tripal_bulk_loader_update($node) { 00438 00439 // Update tripal_bulk_loader 00440 $sql = "UPDATE {tripal_bulk_loader} SET nid = %d, loader_name = '%s', template_id = %d, file = '%s', file_has_header = '%s', keep_track_inserted = %d WHERE nid = %d"; 00441 db_query($sql, $node->nid, $node->loader_name, $node->template_id, $node->file, $node->has_header, $node->keep_track_inserted, $node->nid); 00442 00443 // Add a job if the user want to load the data 00444 global $user; 00445 if ($node->job) { 00446 $job_args[0] =$node->loader_name; 00447 $job_args[1] = $node->template_id; 00448 $job_args[2] = $node->file; 00449 if (is_readable($node->file)) { 00450 $fname = preg_replace("/.*\/(.*)/", "$1", $node->file); 00451 tripal_add_job("Bulk Load: $fname", 'tripal_bulk_loader', 'tripal_bulk_loader_load_data', $job_args, $user->uid); 00452 } 00453 else { 00454 drupal_set_message(t("Can not open %file. Job not scheduled.", array('%file' => $node->file))); 00455 } 00456 } 00457 00458 } 00459 00461 00465 function tripal_bulk_loader_preprocess_tripal_bulk_loader_template(&$variables) { 00466 00467 $sql = "SELECT * FROM {tripal_bulk_loader_template} WHERE template_id=%d"; 00468 $template = db_fetch_object(db_query($sql, $variables['template_id'])); 00469 $template->template_array = unserialize($template->template_array); 00470 $variables['template'] = $template; 00471 00472 } 00473 00493 function tripal_bulk_loader_progess_file_get_progress($job_id, $update_progress = TRUE) { 00494 $filename = '/tmp/tripal_bulk_loader_progress-' . $job_id . '.out'; 00495 if (!file_exists($filename)) { 00496 return (object) array(); 00497 } 00498 00499 $num_lines = trim(`wc --lines < $filename`); 00500 $num_records = trim(`grep -o "." $filename | wc --lines`); 00501 00502 $job = db_fetch_object(db_query("SELECT j.*, b.file, b.file_has_header, c.num as num_constant_sets 00503 FROM {tripal_jobs} j 00504 LEFT JOIN {tripal_bulk_loader} b ON b.job_id=j.job_id 00505 LEFT JOIN ( 00506 SELECT nid, count(distinct(group_id)) as num 00507 FROM {tripal_bulk_loader_constants} 00508 GROUP BY nid 00509 ) c ON c.nid=b.nid 00510 WHERE j.job_id=%d", $job_id)); 00511 if ($job->num_constant_sets) { 00512 $num_constant_sets_loaded = round($job->progress / (100 / $job->num_constant_sets), 4); 00513 00514 // If the next constant set has started loading 00515 if ($job->num_constant_sets != $num_constant_sets_loaded) { 00516 00517 // total lines in input file 00518 $total_lines = trim(`wc --lines < $job->file`); 00519 if ($job->file_has_header) { 00520 $total_lines--; 00521 } 00522 00523 // percent of the current constant set loaded 00524 $percent = round($num_lines/$total_lines * 100, 2); 00525 00526 // percent of the total job = (<# fully loaded constant sets> * 100 ) 00527 // + <percent of current constant set> 00528 // / <total number of constant sets> 00529 $total_percent = (($num_constant_sets_loaded * 100) + $percent) / $job->num_constant_sets; 00530 00531 // update the progress of the job 00532 if ($update_progress AND ($percent != 0 OR $percent != 100)) { 00533 tripal_job_set_progress($job_id, round($total_percent, 0)); 00534 } 00535 } 00536 } 00537 00538 return (object) array( 00539 'num_lines' => $num_lines, 00540 'total_lines' => $total_lines, 00541 'percent_file' => $percent, 00542 'num_constant_sets_loaded' => $num_constant_sets_loaded, 00543 'total_percent' => $total_percent, 00544 'num_records' => $num_records 00545 ); 00546 } 00547 00560 function tripal_bulk_loader_job_describe_args($callback, $args) { 00561 00562 $new_args = array(); 00563 if ($callback == 'tripal_bulk_loader_load_data') { 00564 //1st arg is the nid for a bulk loader node 00565 $node = node_load($args[0]); 00566 $new_args['Bulk Loading Job'] = l($node->title, 'node/' . $args[0]); 00567 return $new_args; 00568 } 00569 00570 } 00571 00576 function tripal_bulk_loader_coder_ignore() { 00577 return array( 00578 'path' => drupal_get_path('module', 'tripal_bulk_loader'), 00579 'line prefix' => drupal_get_path('module', 'tripal_bulk_loader'), 00580 ); 00581 }