From: Holger Amann <keeney@fehu.org>
Date: Thu, 21 Jan 2010 21:05:40 +0000 (+0800)
Subject: calculate ETA and write more output messages
X-Git-Url: http://git.neszt.hu/?a=commitdiff_plain;h=76d7bf3990b168ac0eceb56bf9403065d2316dc5;p=mysql2postgres

calculate ETA and write more output messages
added argument for mysql port
---

diff --git a/README b/README
index 46d0b1a..1d9e66b 100644
--- a/README
+++ b/README
@@ -6,19 +6,17 @@ It can translate now most data types and indexes, but if you experience some pro
 to contact me, I'll help you.
 
 
-Stronly required help to make params of databases not hardcoded.
+Strongly required help to make params of databases not hardcoded.
 
 
 
 After launching tool, You can see something like..
 
-Loading invites: ....*.. 7046 (0min 1s)
-Loading looks: . 1455 (0min 1s)
-Loading message_stats: ....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*... 113627 (0min 10s)
-Loading messages: ....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*....*.. 472456 (2min 25s)
-Loading metadata: ....*...
-
-
-
-Point (.) means, that 1000 of records have been read from MySQL. 
-Star (*) means, that 5000 of records have been dumped into PostgreSQL (closeinput has been called).
\ No newline at end of file
+Creating table friendships...
+Created table friendships
+Loading friendships...
+620000 of 638779 rows loaded. [ETA: 2010/01/21 21:32 (00h:00m:01s)]
+638779 rows loaded in 1min 3s
+Indexing table friendships...
+Indexed table friendships
+Table creation 0 min, loading 1 min, indexing 0 min, total 1 min
diff --git a/mysql2psql b/mysql2psql
index 6cc4f5c..9fdb971 100755
--- a/mysql2psql
+++ b/mysql2psql
@@ -118,6 +118,12 @@ class MysqlReader
       end
     end
     
+    def count_rows
+      @reader.mysql.query("SELECT COUNT(*) FROM `#{name}`")  do |res|
+        return res.fetch_row[0].to_i
+      end
+    end
+    
     def has_id?
       !!columns.find {|col| col[:name] == "id"} 
     end
@@ -136,7 +142,7 @@ class MysqlReader
   end
   
   def connect
-    @mysql = Mysql.connect(@host, @user, @passwd, @db, @sock, @flag)
+    @mysql = Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag)
     @mysql.query("SET NAMES utf8")
     @mysql.query("SET SESSION query_cache_type = OFF")
   end
@@ -146,8 +152,8 @@ class MysqlReader
     connect
   end
   
-  def initialize(host = nil, user = nil, passwd = nil, db = nil, sock = nil, flag = nil)
-    @host, @user, @passwd, @db, @sock, @flag = host, user, passwd, db, sock, flag
+  def initialize(host = nil, user = nil, passwd = nil, db = nil, port = nil, sock = nil, flag = nil)
+    @host, @user, @passwd, @db, @port, @sock, @flag = host, user, passwd, db, port, sock, flag
     connect
   end
   
@@ -408,6 +414,7 @@ class PostgresDbWriter < PostgresWriter
   end
   
   def write_table(table)
+    puts "Creating table #{table.name}..."
     primary_keys = []
     primary_key = nil
     maxval = nil
@@ -458,6 +465,7 @@ class PostgresDbWriter < PostgresWriter
   end
   
   def write_indexes(table)
+    puts "Indexing table #{table.name}..."
     if primary_index = table.indexes.find {|index| index[:primary]}
       @conn.exec("ALTER TABLE #{PGconn.quote_ident(table.name)} ADD CONSTRAINT \"#{table.name}_pkey\" PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})")
     end
@@ -493,11 +501,23 @@ class PostgresDbWriter < PostgresWriter
     end
   end
   
+  def format_eta (t)
+    t = t.to_i
+    sec = t % 60
+    min = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02dh:%02dm:%02ds", hour, min, sec)
+  end
+  
   def write_contents(table, reader)
     _time1 = Time.now
     copy_line = "COPY #{PGconn.quote_ident(table.name)} (#{table.columns.map {|column| PGconn.quote_ident(column[:name])}.join(", ")}) FROM stdin;"
     @conn.exec(copy_line)
-    print "Loading #{table.name}: "
+    puts "Counting rows of #{table.name}... "
+    STDOUT.flush
+    puts "Rows counted"
+    rowcount = table.count_rows
+    puts "Loading #{table.name...}"
     STDOUT.flush
     _counter = reader.paginated_read(table, 1000) do |row, counter|
       line = []
@@ -529,19 +549,24 @@ class PostgresDbWriter < PostgresWriter
         end
       end
       @conn.putline(row.join("\t") + "\n")
+       
+      if counter != 0 && counter % 20000 == 0
+        elapsedTime = Time.now - _time1
+        eta = elapsedTime * rowcount / counter - elapsedTime
+        etaf = self.format_eta(eta)
+        etatimef = (Time.now + eta).strftime("%Y/%m/%d %H:%M")
+        printf "\r#{counter} of #{rowcount} rows loaded. [ETA: #{etatimef} (#{etaf})]"
+        STDOUT.flush
+      end
       
       if counter % 5000 == 0
-        print "*"
-        STDOUT.flush
         @conn.endcopy
         @conn.exec(copy_line)
-      elsif counter % 1000 == 0
-        print "."
-        STDOUT.flush
       end
+       
     end
     _time2 = Time.now
-    puts " #{_counter} (#{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s)"
+    puts "\n#{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s"
 #    @conn.putline(".\n")
     @conn.endcopy
   end
@@ -549,6 +574,7 @@ class PostgresDbWriter < PostgresWriter
   def close
     @conn.close
   end
+
 end
  
  
@@ -595,9 +621,9 @@ class Converter
 end
 
 
-reader = MysqlReader.new('localhost', 'root', nil, 'prophotos')
-#writer = PostgresFileWriter.new($ARGV[2] || "output.sql")
-writer = PostgresDbWriter.new('localhost', 'prophotos', '123', 'prophotos_development:old')
-converter = Converter.new(reader, writer, :only_tables => %w(articles images))
+reader = MysqlReader.new('127.0.0.1', 'mysql_user', 'secretpassword', 'sourcedatabasename', 3306)
+#writer = PostgresFileWriter.new(ARGV[0] || "users.sql")
+writer = PostgresDbWriter.new('127.0.0.1', 'postgres_user', 'secretpassword', 'destdatabasename')
+converter = Converter.new(reader, writer, :only_tables => %w(users messages))
 converter.convert