From 0034b46f79a3a182fce4436915635d49db591fa3 Mon Sep 17 00:00:00 2001 From: "Scott J. Pearson" Date: Wed, 2 Jul 2025 12:39:58 -0500 Subject: [PATCH 1/3] Changes to allow files to be synced --- APISyncExternalModule.php | 337 ++++++++++++++++++++++++++++++++++++-- config.json | 13 +- 2 files changed, 334 insertions(+), 16 deletions(-) diff --git a/APISyncExternalModule.php b/APISyncExternalModule.php index 8534346..aaa8895 100644 --- a/APISyncExternalModule.php +++ b/APISyncExternalModule.php @@ -292,9 +292,8 @@ private function export($servers){ $allFieldsBatchBuilder = new BatchBuilder($this->getExportBatchSize()); $latestLogId = $this->getLatestLogId(); - $allRecordsIds = array_column(json_decode( - REDCap::getData($this->getProjectId(), - 'json', + $allRecordsIds = array_column(REDCap::getData($this->getProjectId(), + 'json-array', null, $recordIdFieldName, null, @@ -310,7 +309,7 @@ private function export($servers){ * To work around this, we may need to store the last sync time of each individual record and use it to "catch up" * with past changes if/when unmatched records begin matching again (likely via a full sync of just those records). */ - ), true), $recordIdFieldName); + ), $recordIdFieldName); $exportAllRecords = $this->getProjectSetting('export-all-records') === true; if($exportAllRecords){ @@ -387,9 +386,9 @@ private function export($servers){ $fields[] = $recordIdFieldName; } - $data = json_decode(REDCap::getData( + $data = REDCap::getData( $this->getProjectId(), - 'json', + 'json-array', $recordIds, $fields, [], @@ -402,7 +401,7 @@ private function export($servers){ false, false, $dateShiftDates - ), true); + ); $subBatchData = []; $subBatchSize = 0; @@ -529,7 +528,8 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat $apiKey = $project['export-api-key']; $args = ['content' => 'record']; - + + $prepped_data = []; if($type === self::UPDATE){ $prepped_data = $this->prepareData($project, $data, $recordIdFieldName); $args['overwriteBehavior'] = 'overwrite'; @@ -548,6 +548,16 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat } $results = $this->apiRequest($url, $apiKey, $args); + if (($type === self::UPDATE) && ($project['export-files'] ?? FALSE)) { + # import is from the perspective of the remote server + $recordIds = []; + foreach ($data as $row) { + if (!in_array($row[$recordIdFieldName], $recordIds)) { + $recordIds[] = $row[$recordIdFieldName]; + } + } + $isSuccessful = $this->transferFiles($url, $apiKey, "import", $recordIds, $project, $prepped_data); + } $this->log( $getProjectExportMessage('Finished'), @@ -881,17 +891,269 @@ function importNextBatch(Progress &$progress){ 'format' => 'json', 'records' => $batch ]); - + $response = $this->prepareData($project, $response, $recordIdFieldName); $stopEarly = $this->importBatch($project, $batchText, $batchSize, $response, $progress); - + if (!$stopEarly && ($project['import-files'] ?? FALSE)) { + # For clarity, export from the remote to import locally + $isSuccessful = $this->transferFiles($url, $apiKey, "export", $batch, $project, $response); + if (!$isSuccessful) { + $stopEarly = TRUE; + } + } + $progress->incrementBatch(); if($progress->getBatchIndex() === count($batches) || $stopEarly){ $progress->finishCurrentProject(); } } + private static function getTempFilename(): string { + do { + $tempFilename = APP_PATH_TEMP.bin2hex(random_bytes(20)); + } while (file_exists($tempFilename)); + return $tempFilename; + } + + private static function getMyCURLFile(array $data, string $recordId, string $recordIdFieldName, $event, string $repeatInstrument, $instance, string $fileFieldName) { + $id = self::getMyFileID($data, $recordId, $recordIdFieldName, $event, $repeatInstrument, $instance, $fileFieldName); + if ($id) { + $fileInfo = \REDCap::getFile($id); + if (isset($fileInfo[0]) && isset($fileInfo[1])) { + $mimeType = $fileInfo[0]; + $fileName = $fileInfo[1]; + $fileContents = $fileInfo[2]; + $tempFilename = self::getTempFilename(); + file_put_contents($tempFilename, $fileContents); + $fileOb = curl_file_create($tempFilename, $mimeType, $fileName); + # clean up of file has to happen after the cURL call + return $fileOb; + } + } + return FALSE; + } + + # accesses the filename in the data. If it's an EDOC ID (i.e., numeric), it transforms it into a filename + private static function getMyFilename(array $data, string $recordId, string $recordIdFieldName, $event, string $repeatInstrument, $instance, string $fileFieldName): string { + $id = self::getMyFileID($data, $recordId, $recordIdFieldName, $event, $repeatInstrument, $instance, $fileFieldName); + if (is_numeric($id)) { + $info = \REDCap::getFile($id); + if (is_array($info) && isset($info[1])) { + return $info[1] ?? $id; + } + } + return $id; + } + + private static function getMyFileID(array $data, string $recordId, string $recordIdFieldName, $event, string $repeatInstrument, $instance, string $fileFieldName): string { + if ($instance === "") { + $instance = 1; + } + foreach ($data as $row) { + if ( + ($row[$recordIdFieldName] == $recordId) + && (($row['redcap_event_name'] ?? "") == ($event ?? "")) + && (($row['redcap_repeat_instrument'] ?? "") == $repeatInstrument) + && (($row['redcap_repeat_instance'] ?? 1) == $instance) + ) { + return $row[$fileFieldName] ?? ""; + } + } + return ""; + } + + private static function makeSuccessfulLocalFileDeleteMessage(string $recordId, string $field): string { + return "Successfully deleted locally in record $recordId with field $field"; + } + + private static function makeSuccessfulRemoteFileDeleteMessage(string $server, string $recordId, string $field): string { + return "Successfully deleted on the remote server $server in record $recordId with field $field"; + } + + private static function makeSuccessfulFileTransferMessage(string $actionOnRemote, string $server, string $recordId, string $field): string { + $toFrom = ($actionOnRemote == "export") ? "from" : "on"; + return "Successfully completed $actionOnRemote $toFrom remote server $server in record $recordId with field $field"; + } + + # returns Boolean of whether it transferred all files successfully + private function transferFiles(string $url, string $apiKey, string $actionOnRemote, array $records, array $project, array $sourceData): bool { + if (!in_array($actionOnRemote, ["import", "export"])) { + throw new \Exception("Invalid action!"); + } + if(empty($project[$this->getPrefixedSettingName('field-list-type')])){ + $fieldList = $this->getCachedProjectSetting($this->getPrefixedSettingName('field-list-all')); + } else { + $fieldList = $project[$this->getPrefixedSettingName('field-list')]; + } + $emptyFieldList = empty($fieldList) || ($fieldList === [NULL]); + + $metadata = $this->getMetadata($this->getProjectId()); + $fileFields = []; + foreach ($metadata as $fieldName => $field) { + if (($field['field_type'] == "file") && ($emptyFieldList || in_array($fieldName, $fieldList))) { + $fileFields[] = $fieldName; + } + } + if (empty($fileFields)) { + return TRUE; + } + + $pid = (int)$this->getProjectId(); + $recordIdFieldName = $this->getRecordIdField(); + if ($actionOnRemote == "export") { + $destinationData = \REDCap::getData($pid, "json-array", $records, array_merge([$recordIdFieldName], $fileFields)); + } else { + $destinationRequest = [ + "content" => "record", + "records" => $records, + "fields" => array_merge([$recordIdFieldName], $fileFields) + ]; + # should not fail: records should already be uploaded, without file info + $destinationData = $this->apiRequest($url, $apiKey, $destinationRequest); + } + foreach ($sourceData as $sourceDataRow) { + foreach ($fileFields as $fileFieldName) { + $recordId = $sourceDataRow[$recordIdFieldName]; + $event = $sourceDataRow['redcap_event_name'] ?? NULL; + $instance = $sourceDataRow['redcap_repeat_instance'] ?? 1; + $repeatInstrument = $sourceDataRow['redcap_repeat_instrument'] ?? ""; + if ($actionOnRemote == "export") { + $sourceFilename = $sourceDataRow[$fileFieldName] ?? ""; + $destinationFilename = self::getMyFilename($destinationData, $recordId, $recordIdFieldName, $event, $repeatInstrument, $instance, $fileFieldName); + } else { + # $actionOnRemote == import + $sourceFilename = self::getMyFilename($sourceData, $recordId, $recordIdFieldName, $event, $repeatInstrument, $instance, $fileFieldName); + $destinationFilename = ""; + foreach ($destinationData as $destinationDataRow) { + if ( + ($destinationDataRow[$recordIdFieldName] == $recordId) + && (($destinationDataRow["redcap_event_name"] ?? "") == ($event ?? "")) + && (($destinationDataRow["redcap_repeat_instance"] ?: 1) == $instance) + && (($destinationDataRow["redcap_repeat_instrument"] ?? "") == $repeatInstrument) + && isset($destinationDataRow[$fileFieldName]) + ) { + $destinationFilename = $destinationDataRow[$fileFieldName]; + break; + } + } + } + if ($sourceFilename != $destinationFilename) { + if (($sourceFilename === "") && ($actionOnRemote == "export")) { + # delete local file + if ($event) { + $uploadRow = [ + $recordIdFieldName => $recordId, + "redcap_event_name" => $event, + "redcap_repeat_instance" => $instance, + $fileFieldName => "" + ]; + } else if ($repeatInstrument) { + $uploadRow = [ + $recordIdFieldName => $recordId, + "redcap_repeat_instrument" => $repeatInstrument, + "redcap_repeat_instance" => $instance, + $fileFieldName => "" + ]; + } else { + $uploadRow = [ + $recordIdFieldName => $recordId, + $fileFieldName => "" + ]; + } + $params = [ + "project_id" => $pid, + "dataFormat" => "json-array", + "data" => [$uploadRow], + "overwriteBehavior" => "overwrite", + "skipFileUploadFields" => FALSE // must be set - REDCap's default is TRUE + ]; + $feedback = \REDCap::saveData($params); + if (!empty($feedback['errors'] ?? [])) { + throw new \Exception("Could not delete $fileFieldName in Record $recordId! ".implode("
\n", $feedback['errors'])); + } else { + $this->log(self::makeSuccessfulLocalFileDeleteMessage($recordId, $fileFieldName)); + continue; + } + } else if (($sourceFilename === "") && ($actionOnRemote == "import")) { + # delete remote file + $requestedAction = "delete"; + } else { + # do requested action to overwrite + $requestedAction = $actionOnRemote; + } + $request = [ + 'content' => 'file', + 'action' => $requestedAction, + 'field' => $fileFieldName, + 'record' => $recordId + ]; + if ($event) { + $request['event'] = $event; + } + if ($instance) { + $request['repeat_instance'] = $instance; + } + if (($requestedAction == "import") && $sourceFilename) { + $curlFileOb = self::getMyCURLFile($sourceData, $recordId, $recordIdFieldName, $event, $repeatInstrument, $instance, $fileFieldName); + if ($curlFileOb) { + $request['file'] = $curlFileOb; + } else { + throw new \Exception("Could not create file to import!"); + } + } + + $response = $this->apiRequest($url, $apiKey, $request); + if (isset($request['file'])) { + # clean up has to happen after the cURL call + unlink($request['file']->getFilename()); + } + + if ($requestedAction == "import") { + if (!$response["success"]) { + return FALSE; + } else { + $this->log(self::makeSuccessfulFileTransferMessage($requestedAction, $url, $recordId, $fileFieldName)); + } + } else if ($requestedAction == "delete") { + if (!$response["success"]) { + return FALSE; + } else { + $this->log(self::makeSuccessfulRemoteFileDeleteMessage($url, $recordId, $fileFieldName)); + } + } else if ($response) { + # $requestedAction == export + $newFilename = $response["filename"]; + if ($newFilename === "") { + throw new \Exception("No filename detected with $fileFieldName in Record $recordId!"); + } + $fileContents = $response["contents"]; + # add the suffix to get the mime type in case $newFilename is not supported, as in before REDCap v13.11.3 + $tempSuffix = pathinfo($newFilename, PATHINFO_EXTENSION); + $tempDot = $tempSuffix ? "." : ""; + $tempFilename = self::getTempFilename().$tempDot.$tempSuffix; + file_put_contents($tempFilename, $fileContents); + $newId = \REDCap::storeFile($tempFilename, $pid, $newFilename); + unlink($tempFilename); + if ($newId > 0) { + $fileSaveSuccessful = \REDCap::addFileToField($newId, $pid, $recordId, $fileFieldName, $event, $instance); + if ($fileSaveSuccessful) { + $this->log(self::makeSuccessfulFileTransferMessage($requestedAction, $url, $recordId, $fileFieldName)); + } else { + throw new \Exception("Error when saving $fileFieldName in Record $recordId!"); + } + } else { + throw new \Exception("Could not save $fileFieldName in Record $recordId!"); + } + } else { + throw new \Exception("No remote server response when importing from $fileFieldName from Record $recordId!"); + } + } + } + } + return TRUE; + } + private function prepareData(&$project, $data, $recordIdFieldName){ // perform translations if configured $this->buildTranslations($project); @@ -1042,8 +1304,8 @@ private function importBatch($project, $batchTextPrefix, $batchSize, $response, $this->log("Importing $batchText (and overwriting matching local records)"); $results = \REDCap::saveData( (int)$this->getProjectId(), - 'json', - json_encode($chunk), + 'json-array', + $chunk, 'overwrite', null, null, @@ -1128,6 +1390,9 @@ private function apiRequest($url, $apiKey, $data){ $domainAndPath = array_pop($parts); $protocol = array_pop($parts); $destinationIsLocalhost = $this->isLocalhost($domainAndPath); + $isFileImport = ($data['action'] === 'import') && ($data['content'] === "file"); + $isFileDelete = ($data['action'] === 'delete') && ($data['content'] === "file"); + $isFileExport = ($data['action'] === 'export') && ($data['content'] === "file"); if( empty($protocol) // Add https if missing @@ -1170,9 +1435,16 @@ private function apiRequest($url, $apiKey, $data){ curl_setopt($ch, CURLOPT_MAXREDIRS, 10); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); curl_setopt($ch, CURLOPT_FRESH_CONNECT, 1); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data, '', '&')); + if (!$isFileImport) { + curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data, '', '&')); + } else { + curl_setopt($ch, CURLOPT_POSTFIELDS, $data); + } curl_setopt($ch, CURLOPT_PROXY, PROXY_HOSTNAME); // If using a proxy curl_setopt($ch, CURLOPT_PROXYUSERPWD, PROXY_USERNAME_PASSWORD); // If using a proxy + if ($isFileExport) { + curl_setopt($ch, CURLOPT_HEADER, true); + } $tries = 0; $sleepTime = 60; @@ -1206,11 +1478,15 @@ private function apiRequest($url, $apiKey, $data){ } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE); $error = curl_error($ch); curl_close($ch); - if(!empty($error)){ + if ($isFileImport || $isFileDelete) { + # to return information is passed back + return ["success" => TRUE]; + } else if(!empty($error)){ throw new Exception("CURL Error $errorNumber: $error"); } else if(empty($output)){ @@ -1225,6 +1501,8 @@ private function apiRequest($url, $apiKey, $data){ && $data['action'] === 'delete' && + is_array($decodedOutput) + && $decodedOutput['error'] === $GLOBALS['lang']['api_131'] . ' ' . $data['records'][0] ){ /** @@ -1242,13 +1520,42 @@ private function apiRequest($url, $apiKey, $data){ } } - if(!$decodedOutput){ + if ($isFileExport && ($decodedOutput === NULL)) { + # cURL returns file contents, not a JSON => place in array + $header = substr($output, 0, $headerSize); + $body = substr($output, $headerSize); + return [ + "filename" => self::getFilenameFromHeader($header), + "mimeType" => self::getMimeTypeFromHeader($header), + "contents" => $body + ]; + } else if($decodedOutput === NULL){ throw new Exception("An unexpected response was returned: $output"); } return $decodedOutput; } + private static function getFilenameFromHeader(string $header): string { + if (preg_match('/Content-Disposition:.*?filename="(.+?)"/i', $header, $matches)) { + return $matches[1]; + } else if (preg_match('/Content-Disposition:.*?filename=([^; ]+)/i', $header, $matches)) { + return rawurldecode($matches[1]); + } else if (preg_match('/Content-Type:\s*[^;\s]+;\sname="(.+?)"/i', $header, $matches)) { + return $matches[1]; + } else if (preg_match('/Content-Type:\s*[^;\s]+;\sname=([^; ]+)"/i', $header, $matches)) { + return rawurldecode($matches[1]); + } + return ""; + } + + private static function getMimeTypeFromHeader(string $header): string { + if (preg_match('/Content-Type:\s*([^;\s]+)/i', $header, $matches)) { + return rawurldecode($matches[1]); + } + return ""; + } + function validateSettings($settings){ $checkNumericSetting = function($settingKey, $settingName, $min, $max) use ($settings) { $values = $settings[$settingKey]; diff --git a/config.json b/config.json index 2006ce9..537f418 100644 --- a/config.json +++ b/config.json @@ -11,7 +11,8 @@ } ], "compatibility": { - "redcap-version-min": "11.2.0" + "php-version-min": "7.3.0", + "redcap-version-min": "12.5.2" }, "permissions": [], "project-settings": [ @@ -134,6 +135,11 @@ } ] }, + { + "key": "export-files", + "name": "Check to export files only when the filename has changed. When exported, any files on the server will be overwritten by the new files. Exporting files can significantly increase the time required to sync a project. The holder of the API token must have Delete Record privileges on the remote server.", + "type": "checkbox" + }, { "key": "export-field-list", "name": "Field List", @@ -280,6 +286,11 @@ "name": "Field List", "type": "field-list", "repeatable": true + }, + { + "key": "import-files", + "name": "Check to import files only when the filename has changed. Files that you have saved will be replaced by any downloaded files. Importing files can significantly increase the amount of time required to sync a project.", + "type": "checkbox" }, { "key": "form-translations", From 4860ceea30aff6d38f93b6ec0a21a0dcbf7e65ae Mon Sep 17 00:00:00 2001 From: "Scott J. Pearson" Date: Thu, 3 Jul 2025 11:30:58 -0500 Subject: [PATCH 2/3] Debugged for longitudinal projects --- APISyncExternalModule.php | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/APISyncExternalModule.php b/APISyncExternalModule.php index aaa8895..3c59516 100644 --- a/APISyncExternalModule.php +++ b/APISyncExternalModule.php @@ -955,7 +955,7 @@ private static function getMyFileID(array $data, string $recordId, string $recor ($row[$recordIdFieldName] == $recordId) && (($row['redcap_event_name'] ?? "") == ($event ?? "")) && (($row['redcap_repeat_instrument'] ?? "") == $repeatInstrument) - && (($row['redcap_repeat_instance'] ?? 1) == $instance) + && (($row['redcap_repeat_instance'] ?: 1) == $instance) ) { return $row[$fileFieldName] ?? ""; } @@ -1016,7 +1016,7 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo foreach ($fileFields as $fileFieldName) { $recordId = $sourceDataRow[$recordIdFieldName]; $event = $sourceDataRow['redcap_event_name'] ?? NULL; - $instance = $sourceDataRow['redcap_repeat_instance'] ?? 1; + $instance = $sourceDataRow['redcap_repeat_instance'] ?: 1; $repeatInstrument = $sourceDataRow['redcap_repeat_instrument'] ?? ""; if ($actionOnRemote == "export") { $sourceFilename = $sourceDataRow[$fileFieldName] ?? ""; @@ -1041,7 +1041,15 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo if ($sourceFilename != $destinationFilename) { if (($sourceFilename === "") && ($actionOnRemote == "export")) { # delete local file - if ($event) { + if ($event && $repeatInstrument) { + $uploadRow = [ + $recordIdFieldName => $recordId, + "redcap_event_name" => $event, + "redcap_repeat_instrument" => $repeatInstrument, + "redcap_repeat_instance" => $instance, + $fileFieldName => "" + ]; + } else if ($event) { $uploadRow = [ $recordIdFieldName => $recordId, "redcap_event_name" => $event, @@ -1136,7 +1144,15 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo $newId = \REDCap::storeFile($tempFilename, $pid, $newFilename); unlink($tempFilename); if ($newId > 0) { - $fileSaveSuccessful = \REDCap::addFileToField($newId, $pid, $recordId, $fileFieldName, $event, $instance); + $eventNum = NULL; + if ($event) { + $Proj = new \Project($pid); // REDCap has to be in a project context + $eventNum = $Proj->getEventIdUsingUniqueEventName($event); + if (!$eventNum) { + throw new \Exception("Could not find an event number for $event!"); + } + } + $fileSaveSuccessful = \REDCap::addFileToField($newId, $pid, $recordId, $fileFieldName, $eventNum, $instance); if ($fileSaveSuccessful) { $this->log(self::makeSuccessfulFileTransferMessage($requestedAction, $url, $recordId, $fileFieldName)); } else { From 40e617d31e0fa6f4d4ee122e2a1eca13b6cf0300 Mon Sep 17 00:00:00 2001 From: "Scott J. Pearson" Date: Thu, 3 Jul 2025 12:01:35 -0500 Subject: [PATCH 3/3] Debugged for a few edge cases --- APISyncExternalModule.php | 14 ++++++++++++-- README.md | 24 ++++++++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/APISyncExternalModule.php b/APISyncExternalModule.php index 3c59516..9b7574c 100644 --- a/APISyncExternalModule.php +++ b/APISyncExternalModule.php @@ -1053,7 +1053,7 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo $uploadRow = [ $recordIdFieldName => $recordId, "redcap_event_name" => $event, - "redcap_repeat_instance" => $instance, + "redcap_repeat_instance" => $sourceDataRow['redcap_repeat_instance'], // blank if not repeating $fileFieldName => "" ]; } else if ($repeatInstrument) { @@ -1076,7 +1076,9 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo "overwriteBehavior" => "overwrite", "skipFileUploadFields" => FALSE // must be set - REDCap's default is TRUE ]; + error_log("Uploading ".json_encode($params)); $feedback = \REDCap::saveData($params); + error_log("Got feedback: ".json_encode($feedback)); if (!empty($feedback['errors'] ?? [])) { throw new \Exception("Could not delete $fileFieldName in Record $recordId! ".implode("
\n", $feedback['errors'])); } else { @@ -1125,7 +1127,8 @@ private function transferFiles(string $url, string $apiKey, string $actionOnRemo } } else if ($requestedAction == "delete") { if (!$response["success"]) { - return FALSE; + $errorMessage = isset($response['error']) ? " ".$response['error'] : ""; + throw new \Exception("Could not delete $fileFieldName on Record $recordId on remote server $url".$errorMessage); } else { $this->log(self::makeSuccessfulRemoteFileDeleteMessage($url, $recordId, $fileFieldName)); } @@ -1500,6 +1503,13 @@ private function apiRequest($url, $apiKey, $data){ curl_close($ch); if ($isFileImport || $isFileDelete) { + $decodedOutput = json_decode($output, true); + if (($decodedOutput !== NULL) && !empty($decodedOutput['error'])) { + return [ + "success" => FALSE, + "error" => $decodedOutput['error'] + ]; + } # to return information is passed back return ["success" => TRUE]; } else if(!empty($error)){ diff --git a/README.md b/README.md index ea4d569..50be15b 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ - - # API Sync Automates exporting/importing to/from remote REDCap servers via the API. The Data Dictionaries for the local and remote projects are expected to be either identical or "compatible". Examples of "compatible" data dictionaries might include the destination having additional fields or dropdown choices that the source doesn't have. In general, any scenario should work with this module that works without error when manually exporting & importing from one project to another. Under the hood, the module is essentially automating a full manual CSV export/import, including reporting any errors you would normally receive due to Data Dictionary differences. Functionality could fairly easily be expanded to support additional scenarios, like automatically syncing the data dictionary as well, or specifying an include/exclude list of fields to sync instead of all fields. - + This module stores API keys for remote systems in the module settings table in local REDCap database. The same level of security applies as would apply to PHI or any other sensitive data stored in the local REDCap project. This means users with design rights on the project, REDCap system administrators, server administrators, and/or database administrators would potentially have access to those API keys (as they would any other data on the local system). ## Form and Event Translations + Version 1.7.0 of the API Sync module introduces the form and event translations feature. This allows administrators to set translations for projects so that imported or exported data will have their form/event names translated upon import/export. These translations are editable via the "Configure Translations" module page under the External Modules header in the REDCap project's sidebar. + ### Configuring Translations for Import Projects + The first column of a configured form/event table should contain the names of the form/event names used in the destination project -- that is, the project that data is being imported to. The following columns should contain form/event names used in source projects. When importing data, the module tries to find form/event names listed in the source columns. If a match is found, the name in the data is translated to the name in the first column. @@ -20,7 +21,9 @@ An example project configured with the following table would convert the form na ![Configuring import translations](/readme/import_forms.PNG) Users can select rows by clicking table cells or select columns by clicking the column name. Selected rows and columns can be removed by clicking '- Remove'. Additional rows and columns can be added by clicking '+ Row' or '+ Column' respectively. The 'Export' and 'Import' buttons export or import CSV. + ### Configuring Translations for Export Projects + Export tables are similar to import tables except they can contain only two columns. The first being the local form/event name and the second column containing the form or event name the module will translate the local form/event name to upon export. For this reason, there is no '+ Column' button and columns cannot be removed. @@ -28,17 +31,22 @@ For this reason, there is no '+ Column' button and columns cannot be removed. In the following example table, the module is configured to convert the 'First Form' form name to 'Their Form Name' and 'Other Form' to 'Their Other Form' before exporting data. ![Configuring export translations](/readme/export_forms.PNG) + ### Note: Form and Event Names in REDCap + REDCap forms and events have two names. A display name and a unique name. The unique name is usually not shown to users but it's how REDCap refers to forms and events internally. You may use either in the 'Configure Translations' tables -- the module can usually determine the correct unique name for a given display name. The module won't be able to determine the unique name in the following cases: - 1. The display name contains only non-Latin characters. +1. The display name contains only non-Latin characters. - In this case, REDCap will generate a unique form name using random characters. The module won't be able to guess the unique form name in this case. + In this case, REDCap will generate a unique form name using random characters. The module won't be able to guess the unique form name in this case. +2. The remote instance of REDCap contains multiple forms/events with the same display name. - 2. The remote instance of REDCap contains multiple forms/events with the same display name. - - In this case, REDCap will append some random characters to make the unique name unique for that form/event. + In this case, REDCap will append some random characters to make the unique name unique for that form/event. Using the display name in the above cases will cause imports/exports to fail. The workaround is to determine the unique names for the forms/events and use those instead. You can determine these unique names, for instance, by exporting the raw data as CSV and finding the names within the exported file. + +### File Sync + +This module will handle files in an import or export if the configuration setting to import/export files is checked in the configuration for each given project. Note that files are only transferred **when a filename as changed**. If a filename stays the same, the sync module will assume that the file has stayed the same. When the source file is deleted, the corresponding file on the destination will be deleted. To delete files, the user with the API token must have Delete Record user rights.