windows7下选用MongoDB实现仓库储存设计_MongoDB_脚本

作者:计算机专家

简单的介绍一下,我使用MongoDB的场景。

我们现在的物联网环境下,有部分数据,采样频率为2000条记录/分钟,这样下来一天24*60*2000=2880000约等于300万条数据,以后必然还会增加。之前数据库使用的是mssql,对于数据库的压力很大,同时又需要保证历史查询的响应速度,这种情况下,在单表中数据量大,同时存在读写操作。不得已采用MongoDB来存储数据。如果使用MongoDB,则至少需要三台机器,两台实现读写分离,一台作为仲裁,每台机器的内存暂时配置在16G,公司小,没办法,据说,使用这个MongoDB需要机器内存最少92G,我没有验证过,但是吃内存是公认的,所以内存绝对要保证,就算保证了,也不一定完全就没有意外发生。我们上面的这些特殊的数据是允许少量的丢失的,这些只是做分析使用的,几个月了,暂时还没出现数据丢失的情况,可能最新版本早就修复了吧,新手使用建议多看下官网上的说明。下面直接奔入主题:

一、安装部署和配置环境

1.安装部署mongo-server

这个时候不要启动,接着配置config文件

2.配置Config文件

dbpath=C:/Program Files/MongoDB/Server/3.4/bin/data/db logpath=C:/Program Files/MongoDB/Server/3.4/bin/data/log/master.log pidfilepath=C:/Program Files/MongoDB/Server/3.4/bin/master.pid directoryperdb=true logappend=true replSet=testrs bind_ip=10.1.5.25port=27016 oplogSize=10000noauth = true storageEngine = wiredTigerwiredTigerCacheSizeGB = 2syncdelay = 30wiredTigerCollectionBlockCompressor = snappy

以上是详细的配置参数,其中路径部分根据需要更改, 这里设置的oplogsize大小为10G,根据业务场景进行调整,另外auth权限为null,因为设置权限会增加服务开销,影响效率,最下面几行是内存引擎,可以控制副本集同步及内存限制,防止内存泄露。

3.启动mongo-server

conf={ "_id" : "testrs", "members" : [ { "_id" : 0, "host" : "10.1.5.25:27016" }, { "_id" : 1, "host" : "10.1.5.26:27016" }, { "_id" : 2, "host" : "10.1.5.27:27016" } ]}rs.initiate

此时副本集集群配置已经完成,然后在命令行中输入:rs.status(),查看副本集状态,需要查看同步情况,可以输入命令:db.serverStatus().

Rs.slaveOk()

6..NET操作mongo

连接设置,请参考个人封装Unitoon.Mongo代码所示。

读写速度:Redis>Mongo>Mssqlserver

可容纳数据量:Mssqlserver~Mongo>Redis

存储数据类型:Mongo>Mssqlserver>Redis

Note:内存持续上升,内部没有内存回收机制,若限制内存 ,则可能出现查询速度变慢,数据丢失等问题,建议优化查询效率,建立索引

Db.test.ensureIndex({"username":1, "age":-1})

强制释放内存命令:db.runCommand({closeAllDatabases:1})

1.基类BaseEntity

namespace UnitoonIot.Mongo{ ///  /// 实体基类,方便生成ObjId ///  [Serializable] [ProtoContract(ImplicitFields = ImplicitFields.AllPublic)] //[ProtoInclude(10, typeof] public class BaseEntity { //[BsonRepresentation] public ObjectId Id { get; set; } ///  /// 数据库名称 ///  public string DbName { get; set; } ///  /// 给对象初值 ///  public BaseEntity() { // this.ObjId = ObjectId.GenerateNewId; //this.Id = ObjectId.NewObjectId; } }}

这里需要注意时间格式,MongoDB默认时间格式为国际时间,所以在写入数据时和读取数据时,时间格式要一致,此例中没有对时间进行特殊处理,由传入的时间格式确定。

2.Repository继承接口IMongoRepository

namespace UnitoonIot.Mongo{ public interface IMongoRepository where TEntity : class { }}

3.MongoRepository

using MongoDB.Driver;using MongoDB.Bson;using System;using System.Collections.Generic;using System.Linq;using System.Linq.Expressions;using System.Text;using System.Threading.Tasks;using MongoDB.Bson.Serialization.Attributes;using MongoDB.Driver.Linq;using System.Configuration;using System.IO;using UnitoonIot.AppSetting; namespace UnitoonIot.Mongo{ public class MongoDb { private static string ConnectionStringHost ; private static string UserName ; private static string Password; private static IMongoDatabase _db = null; private static readonly object LockHelper = new object(); ///  /// mongodb初始化 ///  public static void Init() { ConnectionStringHost = "10.1.5.24:27016,10.1.5.24:27016,10.1.5.26:27017"; //AppSettings.GetConfigValue;//"10.1.5.24:27016"; UserName = AppSettings.GetConfigValue; Password = AppSettings.GetConfigValue; } static MongoDb() { } public static IMongoDatabase GetDb(string dbName,string options=null) { if  return _db; lock  { if  return _db; var database = dbName; var userName = UserName; var password = Password; var authentication = string.Empty; var host = string.Empty; if (!string.IsNullOrWhiteSpace { authentication = string.Concat(userName, ':', password, '@'); } if (!string.IsNullOrEmpty && !options.StartsWith { options = string.Concat; } host = string.IsNullOrEmpty ? "localhost" : ConnectionStringHost; database = database ?? "testdb"; //mongodb://[username:password@]host1[:port1][,host2[:port2],…[,hostN[:portN]]][/[database][?options]] var conString = options!=null? $"mongodb://{authentication}{host}/{database}{options}" : $"mongodb://{authentication}{host}/{database}"; var url = new MongoUrl; var mcs = MongoClientSettings.FromUrl; mcs.MaxConnectionLifeTime = TimeSpan.FromMilliseconds; var client = new MongoClient; _db = client.GetDatabase; } return _db; } } ///  /// MongoDb 数据库操作类 ///  public class MongoRepository: IMongoRepository where T : BaseEntity { #region readonly field ///  /// 表名 ///  private readonly IMongoCollection _collection = null; ///  /// 数据库对象 ///  private readonly IMongoDatabase _database; #endregion ///  /// 构造函数 ///  public MongoRepository() { this._database = MongoDb.GetDb(Activator.CreateInstance().DbName, "readPreference =secondaryPreferred ");//primaryPreferred/secondaryPreferred/nearest _collection = _database.GetCollection; } #region 增加 ///  /// 插入对象 ///  /// 插入的对象 public virtual T Insert { // var flag = ObjectId.GenerateNewId.GetProperty.SetValue; //t.Time = DateTime.Now; _collection.InsertOne; return t; } ///  /// 批量插入 ///  /// 要插入的对象集合 public virtual IEnumerable InsertBatch { _collection.InsertMany; return ts; } ///  /// 插入对象 ///  /// 插入的对象 public virtual void InsertAsync { //var flag = ObjectId.GenerateNewId.GetProperty.SetValue; // t.Time = DateTime.Now; _collection.InsertOneAsync; } ///  /// 批量插入 ///  /// 要插入的对象集合 public virtual void InsertBatchAsync { _collection.InsertManyAsync; } #endregion #region 删除 ///  /// 删除 ///  ///  public virtual long Delete { var filter = Builders.Filter.Eq; var result = _collection.DeleteOne; return result.DeletedCount; } ///  /// 删除 ///  ///  public virtual void DeleteAsync { var filter = Builders.Filter.Eq; _collection.DeleteOneAsync; } ///  /// 按条件表达式删除 ///  /// 条件表达式 ///  public virtual long Delete(Expression> predicate) { var result = _collection.DeleteOne; return result.DeletedCount; } ///  /// 按条件表达式删除 ///  /// 条件表达式 ///  public virtual void DeleteAsync(Expression> predicate) { _collection.DeleteOneAsync; } ///  /// 按条件表达式批量删除 ///  /// 条件表达式 ///  public virtual long DeleteBatch(Expression> predicate) { var result = _collection.DeleteMany; return result.DeletedCount; } ///  /// 按条件表达式批量删除 ///  /// 条件表达式 ///  public virtual void DeleteBatchAsync(Expression> predicate) { _collection.DeleteManyAsync; } ///  /// 按检索条件删除 /// 建议用Builders构建复杂的查询条件 ///  /// 条件 ///  public virtual long Delete(FilterDefinition filter) { var result = _collection.DeleteOne; return result.DeletedCount; } ///  /// 按检索条件删除 /// 建议用Builders构建复杂的查询条件 ///  /// 条件 ///  public virtual void DeleteAsync(FilterDefinition filter) { _collection.DeleteOneAsync; } #endregion #region 修改 ///  /// 修改 ///  ///  public virtual long Update { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq; var update = _collection.ReplaceOne(filter, t, new UpdateOptions() { IsUpsert = true }); return update.ModifiedCount; } ///  /// 修改 ///  ///  public virtual void UpdateAsync { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq; _collection.ReplaceOneAsync(filter, t, new UpdateOptions() { IsUpsert = true }); } ///  /// 用新对象替换新文档 ///  /// 查询条件 /// 对象 /// 修改影响文档数 public virtual long Update(Expression> filter, T t) { var update = _collection.ReplaceOne(filter, t, new UpdateOptions() { IsUpsert = true }); return update.ModifiedCount; } ///  /// 用新对象替换新文档 ///  /// 查询条件 /// 对象 /// 修改影响文档数 public virtual long Update(FilterDefinition filter, T t) { var update = _collection.ReplaceOne(filter, t, new UpdateOptions() { IsUpsert = true }); return update.ModifiedCount; } ///  /// 用新对象替换新文档 ///  /// 查询条件 /// 对象 /// 修改影响文档数 public virtual void UpdateAsync(Expression> filter, T t) { _collection.ReplaceOneAsync(filter, t, new UpdateOptions() { IsUpsert = true }); } ///  /// 用新对象替换新文档 ///  /// 查询条件 /// 对象 /// 修改影响文档数 public virtual void UpdateAsync(FilterDefinition filter, T t) { _collection.ReplaceOneAsync(filter, t, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据Id和条件文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 对象Id /// 修改影响文档数 public virtual long Update(string id, UpdateDefinition update) { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq("Id", new ObjectId; var result = _collection.UpdateOne(filter, update, new UpdateOptions() { IsUpsert = true }); return result.ModifiedCount; } ///  /// 根据Id和条件文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 对象Id /// 修改影响文档数 public virtual void UpdateAsync(string id, UpdateDefinition update) { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq("Id", new ObjectId; _collection.UpdateOneAsync(filter, update, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据条件修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual void Update(UpdateDefinition update,Expression> filter) { _collection.UpdateOne(filter, update, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据条件修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual long Update(UpdateDefinition update, FilterDefinition filter) { var result = _collection.UpdateOne(filter, update, new UpdateOptions() { IsUpsert = true }); return result.ModifiedCount; } ///  /// 根据条件修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual void UpdateAsync(UpdateDefinition update, Expression> filter) { _collection.UpdateOneAsync(filter, update, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据条件修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual void UpdateAsync(UpdateDefinition update, FilterDefinition filter) { _collection.UpdateOneAsync(filter, update, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据条件批量修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual long UpdateBatch(UpdateDefinition update, Expression> filter) { var result = _collection.UpdateMany(filter, update, new UpdateOptions() { IsUpsert = true }); return result.ModifiedCount; } ///  /// 根据条件批量修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual long UpdateBatch(UpdateDefinition update, FilterDefinition filter) { var result = _collection.UpdateMany(filter, update, new UpdateOptions() { IsUpsert = true }); return result.ModifiedCount; } ///  /// 根据条件批量修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual void UpdateBatchAsync(UpdateDefinition update, Expression> filter) { _collection.UpdateManyAsync(filter, update, new UpdateOptions() { IsUpsert = true }); } ///  /// 根据条件批量修改文档 ///  /// 修改条件-形如:Builders/.Update.Set /// 查询条件Builders/.Filter.Eq /// 修改影响文档数 public virtual void UpdateBatchAsync(UpdateDefinition update, FilterDefinition filter) { _collection.UpdateManyAsync(filter, update, new UpdateOptions() { IsUpsert = true }); } #endregion #region 查询 #region GetCollection ///  /// 获取操作对象的IMongoCollection集合,强类型对象集合 ///  ///  public virtual IMongoCollection GetCollection() { return _database.GetCollection; } #endregion #region GetSingle ///  /// 查询数据库,检查是否存在指定ID的对象 ///  /// 对象的ID值 /// 存在则返回指定的对象,否则返回Null public virtual T GetById { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq("Id", new ObjectId; var data = _collection.Find.FirstOrDefault(); return data; } ///  /// 查询数据库,检查是否存在指定ID的对象 ///  /// 对象的ID值 /// 存在则返回指定的对象,否则返回Null public virtual async Task GetAsyncById { var filterBuilder = Builders.Filter; var filter = filterBuilder.Eq("Id", new ObjectId; var data = await _collection.FindAsync; return await data.SingleOrDefaultAsync(); } ///  /// 查询数据 ///  /// 过滤条件 ///  public virtual T Get(FilterDefinition filter) { return _collection.Find.FirstOrDefault(); } ///  /// 查询数据 ///  /// 条件表达式 ///  public virtual T Get(Expression> filter) { return _collection.Find.FirstOrDefault(); } ///  /// 查询数据 ///  /// 过滤条件 ///  public virtual async Task GetAsync(FilterDefinition filter) { var data = await _collection.FindAsync; return await data.SingleOrDefaultAsync(); } ///  /// 查询数据 ///  /// 条件表达式 ///  public virtual async Task GetAsync(Expression> filter) { var data = await _collection.FindAsync; return await data.SingleOrDefaultAsync(); } #endregion #region GetMany ///  /// 查询部分数据 ///  /// 过滤条件 ///  public virtual IEnumerable GetMany(FilterDefinition filter) { return _collection.Find.ToEnumerable(); } ///  /// 查询部分数据 ///  /// 条件表达式 ///  public virtual IEnumerable GetMany(Expression> filter) { //return _collection.AsQueryable.ToList(); //return _collection.AsQueryable; return _collection.Find.ToEnumerable; } ///  /// 查询部分数据 ///  /// 过滤条件 ///  public virtual async Task> GetManyAsync(FilterDefinition filter) { var data = await _collection.FindAsync; return await data.ToListAsync(); } ///  /// 查询部分数据 ///  /// 过滤条件 ///  public virtual async Task> GetManyAsync(Expression> filter) { var data = await _collection.FindAsync; return await data.ToListAsync(); } #endregion #region GetAll ///  /// 查询所有记录,复杂查询直接用Linq处理 ///  /// 要查询的对象 public virtual IEnumerable GetAll() { var data = _collection.AsQueryable(); return data.ToEnumerable(); } ///  /// 查询所有记录,复杂查询直接用Linq处理 ///  /// 要查询的对象 public virtual async Task> GetAllAsync() { var data = _collection.AsQueryable(); return await data.ToListAsync(); } ///  /// 查询所有记录,复杂查询直接用Linq处理 ///  /// 要查询的对象 public virtual IQueryable GetAllQueryable() { return _collection.AsQueryable(); } #endregion #region MapReduce ///  /// MapReduce ///  /// 返回一个List列表数据 public IEnumerable GetMap(BsonJavaScript map,BsonJavaScript reduce) { return _collection.MapReduce; } #endregion #endregion } }

本文由杏彩发布,转载请注明来源

关键词: