using System; using System.Collections; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Text; using System.Threading; using System.Collections.Concurrent; using System.Threading.Tasks; namespace Tango.CSV { /// /// Represents a component for reading and writing CSV files. /// /// public class CsvFile : IDisposable { protected bool _is_disposed; internal protected Stream BaseStream; protected static DateTime DateTimeZero = new DateTime(); /// /// Initializes the class. /// static CsvFile() { DefaultCsvDefinition = new CsvDefinition { EndOfLine = "\r\n", FieldSeparator = ',', TextQualifier = '"' }; UseLambdas = true; UseTasks = false; FastIndexOfAny = true; } /// /// Gets or sets the default CSV definition. /// public static CsvDefinition DefaultCsvDefinition { get; set; } /// /// Gets or sets a value indicating whether [use lambdas]. /// public static bool UseLambdas { get; set; } /// /// Gets or sets a value indicating whether [use tasks]. /// public static bool UseTasks { get; set; } /// /// Gets or sets a value indicating whether [fast index of any]. /// public static bool FastIndexOfAny { get; set; } /// /// Reads the specified CSV source. /// /// /// The CSV source. /// public static IEnumerable Read(CsvSource csvSource) where T : new() { var csvFileReader = new CsvFileReader(csvSource); return (IEnumerable)csvFileReader; } /// /// Gets the columns. /// /// /// The CSV source. /// public static IEnumerable GetColumns(CsvSource csvSource) where T : new() { var csvFileReader = new CsvFileReader(csvSource); return csvFileReader.Columns; } /// /// Gets the field separator. /// /// /// The field separator. /// public char FieldSeparator { get; private set; } /// /// Gets the text qualifier. /// /// /// The text qualifier. /// public char TextQualifier { get; private set; } /// /// Gets the columns. /// /// /// The columns. /// public IEnumerable Columns { get; protected set; } /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// public void Dispose() { Dispose(true); } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool disposing) { // overriden in derived classes } } /// /// Represents a component for reading and writing CSV files from and to a collection of objects. /// /// /// public class CsvFile : CsvFile { private readonly char fieldSeparator; private readonly string fieldSeparatorAsString; private readonly char[] invalidCharsInFields; private readonly StreamWriter streamWriter; private readonly char textQualifier; private readonly String[] columns; private Func[] getters; readonly bool[] isInvalidCharInFields; private int linesToWrite; private readonly BlockingCollection csvLinesToWrite = new BlockingCollection(5000); private readonly Thread writeCsvLinesTask; private Task addAsyncTask; /// /// Initializes a new instance of the class. /// /// The CSV destination. public CsvFile(CsvDestination csvDestination) : this(csvDestination, null) { } /// /// Initializes a new instance of the class. /// public CsvFile() { } /// /// Initializes a new instance of the class. /// /// The CSV destination. /// The CSV definition. public CsvFile(CsvDestination csvDestination, CsvDefinition csvDefinition) { if (csvDefinition == null) csvDefinition = DefaultCsvDefinition; this.columns = (csvDefinition.Columns ?? InferColumns(typeof(T))).ToArray(); this.fieldSeparator = csvDefinition.FieldSeparator; this.fieldSeparatorAsString = this.fieldSeparator.ToString(CultureInfo.InvariantCulture); this.textQualifier = csvDefinition.TextQualifier; this.streamWriter = csvDestination.StreamWriter; this.invalidCharsInFields = new[] { '\r', '\n', this.textQualifier, this.fieldSeparator }; this.isInvalidCharInFields = new bool[256]; foreach (var c in this.invalidCharsInFields) { this.isInvalidCharInFields[c] = true; } this.WriteHeader(); this.CreateGetters(); if (CsvFile.UseTasks) { writeCsvLinesTask = new Thread((o) => this.WriteCsvLines()); writeCsvLinesTask.Start(); } this.addAsyncTask = Task.Factory.StartNew(() => { }); } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected override void Dispose(bool disposing) { if (disposing) { _is_disposed = true; // free managed resources addAsyncTask.Wait(); if (csvLinesToWrite != null) { csvLinesToWrite.CompleteAdding(); } if (writeCsvLinesTask != null) writeCsvLinesTask.Join(); this.streamWriter.Close(); } } /// /// Infers the columns. /// /// Type of the record. /// protected static IEnumerable InferColumns(Type recordType) { var columns = recordType .GetProperties(BindingFlags.Public | BindingFlags.Instance) .Where(pi => pi.GetIndexParameters().Length == 0 && pi.GetSetMethod() != null && !Attribute.IsDefined(pi, typeof(CsvIgnoreAttribute))) .Select(pi => pi.Name) .Concat(recordType .GetFields(BindingFlags.Public | BindingFlags.Instance) .Where(fi => !Attribute.IsDefined(fi, typeof(CsvIgnoreAttribute))) .Select(fi => fi.Name)) .ToList(); return columns; } /// /// Writes the CSV lines. /// private void WriteCsvLines() { int written = 0; foreach (var csvLine in csvLinesToWrite.GetConsumingEnumerable()) { this.streamWriter.WriteLine(csvLine); written++; } Interlocked.Add(ref this.linesToWrite, -written); } /// /// Appends the specified record. /// /// The record. public void Append(T record) { if (_is_disposed) { return; } if (CsvFile.UseTasks) { var linesWaiting = Interlocked.Increment(ref this.linesToWrite); Action addRecord = (t) => { if (!_is_disposed) { var csvLine = this.ToCsv(record); this.csvLinesToWrite.Add(csvLine); } }; if (linesWaiting < 10000) this.addAsyncTask = this.addAsyncTask.ContinueWith(addRecord); else addRecord(null); } else { var csvLine = this.ToCsv(record); this.streamWriter.WriteLine(csvLine); } } /// /// Finds the getter. /// /// The c. /// if set to true [static member]. /// private static Func FindGetter(string c, int index, bool staticMember) { var flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.IgnoreCase | (staticMember ? BindingFlags.Static : BindingFlags.Instance); Func func = null; PropertyInfo pi = null; pi = typeof(T).GetProperty(c); if (pi == null) //Then try get by column index, { pi = typeof(T).GetProperties()[index]; pi = typeof(T).GetProperties()[index]; //Workaround for some weired problem that makes the program not take the value. } if (CsvFile.UseLambdas) { Expression expr = null; ParameterExpression parameter = Expression.Parameter(typeof(T), "r"); Type type = null; if (pi != null) { type = pi.PropertyType; expr = Expression.Property(parameter, pi.Name); } if (expr != null) { Expression> lambda; if (type.IsValueType) { lambda = Expression.Lambda>(Expression.TypeAs(expr, typeof(object)), parameter); } else { lambda = Expression.Lambda>(expr, parameter); } func = lambda.Compile(); } } else { if (pi != null) func = o => pi.GetValue(o, null); } return func; } /// /// Creates the getters. /// private void CreateGetters() { var list = new List>(); for (int i = 0; i < columns.Length; i++) { Func func = null; var propertyName = (columns[i].IndexOf(' ') < 0 ? columns[i] : columns[i].Replace(" ", "")); func = FindGetter(columns[i], i, false) ?? FindGetter(columns[i], i, true); list.Add(func); } this.getters = list.ToArray(); } /// /// To the CSV. /// /// The record. /// /// Cannot be null;record private string ToCsv(T record) { if (record == null) throw new ArgumentException("Cannot be null", "record"); string[] csvStrings = new string[getters.Length]; for (int i = 0; i < getters.Length; i++) { var getter = getters[i]; object fieldValue = getter == null ? null : getter(record); csvStrings[i] = this.ToCsvString(fieldValue); } return string.Join(this.fieldSeparatorAsString, csvStrings); } /// /// To the CSV string. /// /// The o. /// private string ToCsvString(object o) { if (o != null) { string valueString = o as string ?? Convert.ToString(o, CultureInfo.CurrentUICulture); if (RequireQuotes(valueString)) { var csvLine = new StringBuilder(); csvLine.Append(this.textQualifier); foreach (char c in valueString) { if (c == this.textQualifier) csvLine.Append(c); // double the double quotes csvLine.Append(c); } csvLine.Append(this.textQualifier); return csvLine.ToString(); } else return valueString; } return string.Empty; } /// /// Requires the quotes. /// /// The value string. /// private bool RequireQuotes(string valueString) { if (CsvFile.FastIndexOfAny) { var len = valueString.Length; for (int i = 0; i < len; i++) { char c = valueString[i]; if (c <= 255 && this.isInvalidCharInFields[c]) return true; } return false; } else { return valueString.IndexOfAny(this.invalidCharsInFields) >= 0; } } /// /// Writes the header. /// private void WriteHeader() { var csvLine = new StringBuilder(); for (int i = 0; i < this.columns.Length; i++) { if (i > 0) csvLine.Append(this.fieldSeparator); csvLine.Append(this.ToCsvString(this.columns[i])); } this.streamWriter.WriteLine(csvLine.ToString()); } } }