在 Node 环境中导入 SQL 转储

Posted 2023-03-07

【中文标题】在 Node 环境中导入 SQL 转储

【英文标题】：Import SQL dump within Node environment

【发布时间】：2016-07-23 08:56:24

【问题描述】：

我想要一个 npm 脚本来创建/配置/等。最后导入一个 SQL 转储。整个创建、配置等都在工作，但是，我无法让导入工作。永远不会插入数据。这是我所拥有的（不要介意嵌套回调，因为它们会变成承诺）：

connection.query(`DROP DATABASE IF EXISTS $config.database;`, err => 
  connection.query(`CREATE DATABASE IF NOT EXISTS $config.database;`, err => 
    connection.query('use DATABASENAME', err => 
      const sqlDumpPath = path.join(__dirname, 'sql-dump/sql-dump.sql');
      connection.query(`SOURCE $sqlDumpPath`, err => 
        connection.end(err => resolve());
      );
    )
  );
);

我还使用 Sequelize (ORM) 尝试了以下操作：

return new Promise(resolve => 
  const sqlDumpPath = path.join(__dirname, 'sql-dump/sql-dump.sql');
  fs.readFile('./sql/dump.sql', 'utf-8', (err, data) => 
    sequelize
      .query(data)
      .then(resolve)
      .catch(console.error);
  );
); 

【参考方案1】：

以下是我使用 migrations 框架设置初始 Sequelized 导入的方法。这里有很多事情要做，但总之我：

在迁移文件夹中找到最新的 sql-dump 使用fs读取文件 将文本拆分为查询 检查它是否是一个有效的查询，如果是，则应用我的数据所需的一些清理 (see related post) 推送一个充满查询的数组 - 我首先通过调用 this.down 来确保数据库是干净的 使用mapSeries（不是map）将所有内容作为promise（建议here）运行

使用sequelize-cli，你可以在你的shell中通过写来创建一个迁移：

sequelize migration:create

您将自动获得在其中输入以下代码的文件。为了执行迁移，您只需编写：

sequelize db:migrate

"use strict";
const promise = require("bluebird");
const fs = require("fs");
const path = require("path");
const assert = require("assert");
const db = require("../api/models"); // To be able to run raw queries
const debug = require("debug")("my_new_api");

// I needed this in order to get some encoding issues straight
const Aring = new RegExp(String.fromCharCode(65533) +
  "\\" + String.fromCharCode(46) + "1,3", "g");
const Auml = new RegExp(String.fromCharCode(65533) +
  String.fromCharCode(44) + "1,3", "g");
const Ouml = new RegExp(String.fromCharCode(65533) +
  String.fromCharCode(45) + "1,3", "g");

module.exports = 
  up: function (queryInterface, Sequelize) 
    // The following section allows me to have multiple sql-files and only use the last dump
    var last_sql;
    for (let fn of fs.readdirSync(__dirname))
      if (fn.match(/\.sql$/))
        fn = path.join(__dirname, fn);
        var stats = fs.statSync(fn);
        if (typeof last_sql === "undefined" ||
            last_sql.stats.mtime < stats.mtime)
          last_sql = 
            filename: fn,
            stats: stats
          ;
        
      
    
    assert(typeof last_sql !== "undefined", "Could not find any valid sql files in " + __dirname);

    // Split file into queries
    var queries = fs.readFileSync(last_sql.filename).toString().split(/;\n/);

    var actions = [
      query: "Running the down section",
      exec: this.down
    ]; // Clean database by calling the down first

    for (let i in queries)
      // Skip empty queries and the character set information in the 40101 section
      //   as this would most likely require a multi-query set-up
      if (queries[i].trim().length == 0 ||
          queries[i].match(new RegExp("/\\*!40101 .+ \\*/")))
        continue;
      

      // The manual fixing of encoding
      let clean_query = queries[i]
        .replace(Aring, "Å")
        .replace(Ouml, "Ö")
        .replace(Auml, "Ä");

      actions.push(
        query: clean_query.substring(0, 200), // We save a short section of the query only for debugging purposes
        exec: () => db.sequelize.query(clean_query)
      );
    

    // The Series is important as the order isn't retained with just map
    return promise.mapSeries(actions, function(item) 
      debug(item.query);

      return item.exec();
    ,  concurrency: 1 );
  ,

  down: function (queryInterface, Sequelize) 
    var tables_2_drop = [
      "items",
      "users",
      "usertypes"
    ];
    var actions = [];
    for (let tbl of tables_2_drop)
      actions.push(
        // The created should be created_at
        exec: () => db.sequelize.query("DROP TABLE IF EXISTS `" + tbl +"`")
      );
    

    return promise.map(actions, function(item) 
      return item.exec();
    ,  concurrency: 1 );/**/
  
;

【讨论】：

这对我很有用，尽管您在查询中替换字符告诉我您在读取转储文件时使用了错误的编码。将 fs.readFileSync(last_sql.filename) 更改为 fs.readFileSync(last_sql.filename, , encoding: "UTF-8")

【参考方案2】：

大致基于 Max Gordon 的 answer，这是我从 NodeJs/Sequelize 运行 mysql Dump 文件的代码：

"use strict";

const fs = require("fs");
const path = require("path");

/**
 * Start off with a MySQL Dump file, import that, and then migrate to the latest version.
 *
 * @param dbName string the name of the database
 * @param mysqlDumpFile string The full path to the file to import as a starting point
 */
module.exports.migrateFromFile = function(dbName, mysqlDumpFile) 
  let sequelize = createSequelize(dbName);
  console.log("Importing from " + mysqlDumpFile + "...");
  let queries = fs.readFileSync(mysqlDumpFile, encoding: "UTF-8").split(";\n");

  console.log("Importing dump file...");

  // Setup the DB to import data in bulk.
  let promise = sequelize.query("set FOREIGN_KEY_CHECKS=0"
  ).then(() => 
    return sequelize.query("set UNIQUE_CHECKS=0");
  ).then(() => 
    return sequelize.query("set SQL_MODE='NO_AUTO_VALUE_ON_ZERO'");
  ).then(() => 
    return sequelize.query("set SQL_NOTES=0");
  );

  console.time("Importing mysql dump");
  for (let query of queries) 
    query = query.trim();
    if (query.length !== 0 && !query.match(/\/\*/)) 
      promise = promise.then(() => 
        console.log("Executing: " + query.substring(0, 100));
        return sequelize.query(query, raw: true);
      )
    
  

  return promise.then(() => 
    console.timeEnd("Importing mysql dump");

    console.log("Migrating the rest of the way...");
    console.time("Migrating after importing mysql dump");
    return exports.migrateUp(dbName); // Run the rest of your migrations
  ).then(() => 
    console.timeEnd("Migrating after importing mysql dump");
  );

;