获取原始数据队列

AppStart/StartHelper.cs

@@ -12,6 +12,7 @@
             int Environment = Library.ConfigurationManager.EnvironmentFlag; //运行环境（1-开发环境，2-生产环境）
             if(Environment == 2)
             {
+                SourceHelper.Instance.Start(); //获取原始数据队列
 
                 // DefaultHelper.Instance.Start(); //队列程序
                 // ......其他队列程序

Models/Push/SourceData.cs

@@ -14,5 +14,6 @@ namespace MySystem.Models.Push
         public string FieldList { get; set; }
         public string TableName { get; set; }
         public string Title { get; set; }
+        public string TargetFieldList { get; set; }
     }
 }

Models/Push/WebCMSEntities.cs

@@ -392,8 +392,8 @@ namespace MySystem.Models.Push
 
                 entity.Property(e => e.FieldList)
                     .HasColumnName("field_list")
-                    .HasColumnType("varchar(500)")
-                    .HasComment("数据字段")
+                    .HasColumnType("mediumtext")
+                    .HasComment("查询数据字段")
                     .HasCharSet("utf8")
                     .HasCollation("utf8_general_ci");
 
@@ -414,6 +414,13 @@ namespace MySystem.Models.Push
                     .HasCharSet("utf8")
                     .HasCollation("utf8_general_ci");
 
+                entity.Property(e => e.TargetFieldList)
+                    .HasColumnName("target_field_list")
+                    .HasColumnType("mediumtext")
+                    .HasComment("目标表字段")
+                    .HasCharSet("utf8")
+                    .HasCollation("utf8_general_ci");
+
                 entity.Property(e => e.Title)
                     .HasColumnName("title")
                     .HasColumnType("varchar(50)")

Util/Queue/SourceHelper.cs

@@ -0,0 +1,146 @@
+using System;
+using System.Linq;
+using System.Threading;
+using Library;
+using LitJson;
+using MySystem.Models.Push;
+
+namespace MySystem
+{   
+    public class SourceHelper
+    {
+        public readonly static SourceHelper Instance = new SourceHelper();
+        private SourceHelper()
+        {
+        }
+
+        public void Start()//启动
+        {
+            Thread thread = new Thread(threadStart);
+            thread.IsBackground = true;
+            thread.Start();
+        }
+
+        private void threadStart()
+        {
+            while (true)
+            {
+                DoSomeThing();
+                Thread.Sleep(1000);
+            }
+        }
+
+        //要执行的方法
+        public void DoSomeThing()
+        {
+            WebCMSEntities db = new WebCMSEntities();
+            var sources = db.SourceData.Where(m => m.Status == 1).ToList();
+            foreach(var source in sources)
+            {
+                string Title = source.Title; //说明
+                string TableName = source.TableName; //数据表
+                string FieldList = source.FieldList; //原始数据字段
+                string TargetFieldList = source.TargetFieldList; //目标数据字段
+                string QueryField = "";
+                JsonData fieldJson = JsonMapper.ToObject(FieldList);
+                for (int i = 0; i < fieldJson.Count; i++)
+                {
+                    JsonData item = fieldJson[i];
+                    QueryField += item["name"].ToString() + ",";
+                }
+
+                //构造抓取数据
+                string sql = "select " + QueryField.TrimEnd(',') + " from " + TableName + " where 1=1";
+                var condiList = db.SourceCondition.Where(m => m.SourceId == source.Id).ToList();
+                foreach(var condi in condiList)
+                {
+                    int QueryCondition = condi.QueryCondition;
+                    if(QueryCondition == 1)
+                    {
+                        sql += " and " + condi.QueryField + "='" + condi.QueryVal + "'";
+                    }
+                    else if(QueryCondition == 2)
+                    {
+                        sql += " and " + condi.QueryField + " like '%" + condi.QueryVal + "%'";
+                    }
+                    else if(QueryCondition == 3)
+                    {
+                        string[] QueryValList = condi.QueryVal.Split('|');
+                        sql += " and " + condi.QueryField + ">=" + QueryValList[0] + " and " + condi.QueryField + "<=" + QueryValList[1] + "";
+                    }
+                    else if(QueryCondition == 4)
+                    {
+                        string[] QueryValList = condi.QueryVal.Split('|');
+                        sql += " and " + condi.QueryField + ">='" + QueryValList[0] + "' and " + condi.QueryField + "<='" + QueryValList[1] + "'";
+                    }
+                    else if(QueryCondition == 5)
+                    {
+                        sql += " and " + condi.QueryField + " in (" + condi.QueryVal + ")";
+                    }
+                    else if(QueryCondition == 6)
+                    {
+                        sql += " and " + condi.QueryField + " in ('" + condi.QueryVal.Replace(",", "','") + "')";
+                    }
+                    else if(QueryCondition == 7)
+                    {
+                        string[] QueryValList = condi.QueryVal.Split(',');
+                        sql += " and (";
+                        int index = 0;
+                        foreach(string QueryVal in QueryValList)
+                        {
+                            index += 1;
+                            sql += condi.QueryField + "=" + QueryVal;
+                            if(index < QueryValList.Length)
+                            {
+                                sql += " or ";
+                            }
+                        }
+                        sql += ")";
+                    }
+                    else
+                    {
+                        sql += " and " + condi.QueryField + "=" + condi.QueryVal;
+                    }
+                }
+
+                //创建原始数据表
+                var sourceEdit = db.SourceData.FirstOrDefault(m => m.Id == source.Id && m.Version == 0);
+                if(sourceEdit != null)
+                {
+                    CreateTable(Title, TableName, TargetFieldList);
+                    sourceEdit.Version = 1;
+                    db.SaveChanges();
+                }
+
+                //插入数据
+                string TargetQueryField = "";
+                JsonData TargetFieldJson = JsonMapper.ToObject(TargetFieldList);
+                for (int i = 0; i < TargetFieldJson.Count; i++)
+                {
+                    JsonData item = TargetFieldJson[i];
+                    TargetQueryField += item["name"].ToString() + ",";
+                }
+                string insertSql = "insert into " + TableName + " (" + TargetQueryField + ")" + sql;
+                CustomerSqlConn.op(insertSql, AppConfig.Base.SqlConnStr);
+            }
+            db.Dispose();
+        }
+        
+        //创建表
+        public void CreateTable(string Title, string tableName, string fieldList)
+        {
+            string sql = "CREATE TABLE `" + tableName + "` (\n";
+            sql += "`id` int(11) NOT NULL AUTO_INCREMENT,\n";
+            sql += "`status` int(11) NOT NULL,\n";
+            JsonData fieldJson = JsonMapper.ToObject(fieldList);
+            for (int i = 0; i < fieldJson.Count; i++)
+            {
+                JsonData item = fieldJson[i];
+                sql += "`" + item["name"].ToString() + "` varchar(" + item["len"].ToString() + ") DEFAULT NULL COMMENT '" + item["title"].ToString() + "',\n";
+            }
+            sql += "PRIMARY KEY (`id`)\n";
+            sql += ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='" + Title + "';";
+            CustomerSqlConn.op(sql, AppConfig.Base.SqlConnStr);
+        }
+    }
+}