4 var FS = require('q-io/fs');
5 var util = require('util');
6 var colors = require('colors');
7 var csv = require('csv');
8 var rest = require('restler-q');
9 var async = require('async');
10 var _ = require('lodash');
11 var argv = require('optimist')
12 .demand(['i', 'c', 'g', 'p', 't', 's'])
15 .alias('g', 'gitlaburl')
16 .alias('p', 'project')
20 .describe('i', 'CSV file exported from Mantis (Example: issues.csv)')
21 .describe('c', 'Configuration file (Example: config.json)')
22 .describe('g', 'GitLab URL hostname (Example: https://gitlab.com)')
23 .describe('p', 'GitLab project name including namespace (Example: mycorp/myproj)')
24 .describe('t', 'An admin user\'s private token (Example: a2r33oczFyQzq53t23Vj)')
25 .describe('s', 'The username performing the import (Example: bob)')
26 .describe('f', 'The first issue # to import (Example: 123)')
29 var inputFile = argv.input;
30 var configFile = argv.config;
31 var fromIssueId = Number(argv.from||0);
32 var gitlabAPIURLBase = argv.gitlaburl + '/api/v4';
33 var gitlabProjectName = argv.project;
34 var gitlabAdminPrivateToken = argv.token;
35 var gitlabSudo = argv.sudo;
39 var promise = getConfig()
40 .then(readMantisIssues)
41 .then((JSON.stringify(gitLab.mantisIssues)))
42 .then(getGitLabProject)
43 .then(getGitLabProjectMembers)
44 .then(mapGitLabUserIds)
45 .then(validateMantisIssues)
46 .then(getGitLabProjectIssues)
47 .then(importGitLabIssues)
50 promise.then(function() {
51 console.log(("Done!").bold.green);
57 * Read and parse config.json file - assigns config
59 function getConfig() {
60 log_progress("Reading configuration...");
61 return FS.read(configFile)
62 .then(function(data) {
63 var config = JSON.parse(data);
65 config.users = _.extend({
68 gl_username: gitlabSudo
72 }).then(function(cfg) {
75 throw new Error('Cannot read config file: ' + configFile);
80 * Read and parse import.csv file - assigns gitLab.mantisIssues
82 function readMantisIssues() {
83 log_progress("Reading Mantis export file...");
84 return FS.read(inputFile, {encoding: 'utf8'}).then(function(data) {
88 csv().from(data, {delimiter: '\t', quote: '~', escape: '~', columns: true})
89 .on('record', function(row, index) { rows.push(row) })
90 .on('end', function(error, data) {
95 .then(function(rows) {
96 _.forEach(rows, function(row) {
97 row.Id = Number(row.Id);
101 rows = _.filter(rows, function(row) {
102 return row.Id >= fromIssueId;
106 return gitLab.mantisIssues = _.sortBy(rows, "Id");
108 throw new Error('Cannot read input file: ' + inputFile + " - " + error);
114 * Fetch project info from GitLab - assigns gitLab.project
116 function getGitLabProject() {
117 log_progress("Fetching project from GitLab...");
118 var url = gitlabAPIURLBase + '/projects';
119 var data = { per_page: 100, private_token: gitlabAdminPrivateToken, sudo: gitlabSudo };
121 return rest.get(url, {data: data})
122 .fail(function(err) {
125 .then(function(result) {
126 gitLab.project = _.find(result, { path_with_namespace : gitlabProjectName }) || null;
128 if (!gitLab.project) {
129 throw new Error('Cannot find GitLab project: ' + gitlabProjectName);
132 return gitLab.project;
134 throw new Error('Cannot get list of projects from gitlab: ' + url);
139 * Fetch project members from GitLab - assigns gitLab.gitlabUsers
141 function getGitLabProjectMembers() {
142 log_progress("getGitLabProjectMembers");
143 var url = gitlabAPIURLBase + '/projects/' + gitLab.project.id + "/members";
144 var data = { per_page: 100, private_token: gitlabAdminPrivateToken, sudo: gitlabSudo };
146 return rest.get(url, {data: data}).then(function(result) {
147 return gitLab.gitlabUsers = result;
149 throw new Error('Cannot get list of users from gitlab: ' + url);
154 * Sets config.users[].gl_id based gitLab.gitlabUsers
156 function mapGitLabUserIds() {
157 var users = config.users,
158 gitlabUsers = gitLab.gitlabUsers;
159 _.forEach(users, function(user) {
160 user.gl_id = (_.find(gitlabUsers, { username: user.gl_username }) || {}).id;
165 * Ensure that Mantise user names in gitLab.mantisIssues have corresponding GitLab user mapping
167 function validateMantisIssues() {
168 log_progress("Validating Mantis Users...");
170 var mantisIssues = gitLab.mantisIssues;
171 var users = config.users;
173 var missingUsernames = [];
175 for (var i = 0; i < mantisIssues.length; i++) {
176 var assignee = mantisIssues[i]["Assigned To"];
178 if (!getUserByMantisUsername(assignee) && missingUsernames.indexOf(assignee) == -1)
179 missingUsernames.push(assignee);
182 for (var i = 0; i < mantisIssues.length; i++) {
183 var reporter = mantisIssues[i].Reporter;
185 if (!getUserByMantisUsername(reporter) && missingUsernames.indexOf(reporter) == -1)
186 missingUsernames.push(reporter);
189 if (missingUsernames.length > 0) {
190 for (var i = 0; i < missingUsernames.length; i++)
191 console.error('Error: Cannot map Mantis user with username: ' + missingUsernames[i]);
193 throw new Error("User Validation Failed");
198 * Import gitLab.mantisIssues into GitLab
201 function importGitLabIssues() {
202 log_progress("Importing Mantis issues into GitLab from #" + fromIssueId + " ...");
203 return _.reduce(gitLab.mantisIssues, function(p, mantisIssue) {
204 return p.then(function() {
205 return importIssue(mantisIssue);
211 function importIssue(mantisIssue) {
212 var issueId = mantisIssue.Id;
213 var title = mantisIssue.Summary;
214 var description = getDescription(mantisIssue);
215 var assignee = getUserByMantisUsername(mantisIssue["Assigned To"]);
216 var milestoneId = '';
217 var labels = getLabels(mantisIssue);
218 var author = getUserByMantisUsername(mantisIssue.Reporter);
220 log_progress("Importing: #" + issueId + " - " + title + " ...");
224 description: description,
225 assignee_id: assignee && assignee.gl_id,
226 milestone_id: milestoneId,
229 private_token: gitlabAdminPrivateToken
232 return getIssue(gitLab.project.id, issueId)
233 .then(function(gitLabIssue) {
234 //console.log(data.title + " -- " + issueId);
235 //console.log(data.title + " -- " + issueId + " -- " + gitLab.gitlabIssues[issueId]);
236 if (gitLab.gitlabIssues[issueId]) {
238 console.log("updating: " + data.title + " (Issueid: " + issueId + ")");
239 return updateIssue(gitLab.project.id, gitLabIssue.iid, _.extend({
240 state_event: isClosed(mantisIssue) ? 'close' : 'reopen'
243 console.log(("#" + gitLabIssue.iid + ": Updated successfully.").green);
247 console.log("inserting: " + data.title + " (Issueid: " + issueId + ")");
248 return insertSkippedIssues(issueId-1)
250 return insertAndCloseIssue(issueId, data, isClosed(mantisIssue));
256 function insertSkippedIssues(issueId) {
257 if (issueId < 1 || gitLab.gitlabIssues[issueId]) {
261 //console.warn(("Skipping Missing Mantis Issue (<= #" + issueId + ") ...").yellow);
262 console.log(("Adding placeholder ...").yellow);
265 title: "-- Platzhalter --",
266 description: "Diese Bugnummer war in Mantis einem anderen Projekt zugeordnet. Dieser Platzhalter sorgt dafuer, dass die Bugnummern in Mantis mit denen in Gitlab uebereinstimmen.",
268 private_token: gitlabAdminPrivateToken
271 return insertAndCloseIssue(issueId, data, true)
273 return insertSkippedIssues(issueId);
277 function insertAndCloseIssue(issueId, data, close, custom) {
279 return insertIssue(gitLab.project.id, data).then(function(issue) {
280 gitLab.gitlabIssues[issue.iid] = issue;
282 return closeIssue(issue, custom && custom(issue)).then(
284 console.log((issue.iid + ': Inserted and closed successfully. #' + issue.iid).green);
286 console.warn((issue.iid + ': Inserted successfully but failed to close. #' + issue.iid).yellow);
290 console.log((issue.iid + ': Inserted successfully. #' + issue.iid).green);
292 console.error((issue.iid + ': Failed to insert.').red, error);
297 * Fetch all existing project issues from GitLab - assigns gitLab.gitlabIssues
299 function getGitLabProjectIssues() {
300 return getRemainingGitLabProjectIssues(0, 100)
301 .then(function(result) {
302 log_progress("Fetched " + result.length + " GitLab issues.");
303 var issues = _.indexBy(result, 'iid');
304 return gitLab.gitlabIssues = issues;
309 * Recursively fetch the remaining issues in the project
313 function getRemainingGitLabProjectIssues(page, per_page) {
314 var from = page * per_page;
315 log_progress("Fetching Project Issues from GitLab [" + (from + 1) + "-" + (from + per_page) + "]...");
316 var url = gitlabAPIURLBase + '/projects/' + gitLab.project.id + "/issues";
322 private_token: gitlabAdminPrivateToken, sudo: gitlabSudo };
324 return rest.get(url, {data: data})
325 .fail(function(err) {
328 .then(function(issues) {
329 if(issues.length < per_page) {
332 return getRemainingGitLabProjectIssues(page+1, per_page)
333 .then(function(remainingIssues) {
334 return issues.concat(remainingIssues);
337 throw new Error('Cannot get list of issues from gitlab: ' + url + " page=" + page);
341 function getUserByMantisUsername(username) {
342 return (username && config.users[username]) || config.users[""] || null;
345 function getDescription(row) {
347 var issueId = row.Id;
349 if (config.mantisUrl) {
350 attributes.push("[Mantis Issue " + issueId + "](" + config.mantisUrl + "/view.php?id=" + issueId + ")");
352 attributes.push("Mantis Issue " + issueId);
355 if (value = row.Reporter) {
356 attributes.push("Reported By: " + value);
360 if (value = row["Assigned To"]) {
361 attributes.push("Assigned To: " + value);
365 if (value = row.Created) {
366 attributes.push("Created: " + value);
369 if (value = row.Updated && value != row.Created) {
370 attributes.push("Updated: " + value);
373 var description = "_" + attributes.join(", ") + "_\n\n";
375 description += row.Description;
377 if (value = row.Info) {
378 description += "\n\n" + value;
381 if (value = row.Notes) {
382 description += "\n\n" + value.split("$$$$").join("\n\n")
385 description = description.replace(/\\n/g, "\n");
386 description = description.replace(/\\t/g, " ");
387 description = description.replace(/``/g, '"');
388 description = description.replace(/''/g, '"');
390 description = description.replace(/\n *----/g, "\n>>>");
392 //description = description.replace(/schnalke/g, "@MSchnalke");
394 Object.keys(config.users).forEach(function(muser) {
396 var gluser = config.users[muser].gl_username;
397 var re = new RegExp(muser, "g");
398 description = description.replace(re, "@" + gluser);
405 function getLabels(row) {
407 var labels = (row.tags || []).slice(0);
409 if(label = config.category_labels[row.Category]) {
413 if(label = config.priority_labels[row.Priority]) {
417 if(label = config.severity_labels[row.Severity]) {
421 return labels.join(",");
424 function isClosed(row) {
425 return config.closed_statuses[row.Status];
428 function getIssue(projectId, issueId) {
429 return Q(gitLab.gitlabIssues[issueId]);
431 //var url = gitlabAPIURLBase + '/projects/' + projectId + '/issues?iid=' + issueId;
432 //var data = { private_token: gitlabAdminPrivateToken, sudo: gitlabSudo };
434 //return rest.get(url, {data: data})
435 // .then(function(issues) {
436 // var issue = issues[0];
438 // throw new Error("Issue not found: " + issueId);
444 function insertIssue(projectId, data) {
445 var url = gitlabAPIURLBase + '/projects/' + projectId + '/issues';
447 return rest.post(url, {data: data})
448 .then(null, function(error) {
449 throw new Error('Failed to insert issue into GitLab: ' + url);
453 function updateIssue(projectId, issueIId, data) {
454 var url = gitlabAPIURLBase + '/projects/' + projectId + '/issues/' + issueIId;
456 return rest.put(url, {data: data})
457 .then(null, function(error) {
458 throw new Error('Failed to update issue in GitLab: ' + url + " " + JSON.stringify(error));
462 function closeIssue(issue, custom) {
463 var url = gitlabAPIURLBase + '/projects/' + issue.project_id + '/issues/' + issue.iid;
464 var data = _.extend({
465 state_event: 'close',
466 private_token: gitlabAdminPrivateToken,
470 return rest.put(url, {data: data})
471 .then(null, function(error) {
472 throw new Error('Failed to close issue in GitLab: ' + url);
477 function log_progress(message) {
478 console.log(message.grey);