Software/Visual_Studio/Tango.CSV/CsvFile.cs


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long *///------------------------------------------------------------------------------
// <auto-generated>
//     This code was generated from a template.
//
//     Manual changes to this file may cause unexpected behavior in your application.
//     Manual changes to this file will be overwritten if the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------

namespace Tango.DAL.Remote.DB
{
    using System;
    using System.Collections.Generic;
    
    public partial class MEDIA_PURPOSES
    {
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors")]
        public MEDIA_PURPOSES()
        {
            this.RMLS = new HashSet<RML>();
        }
    
        public int ID { get; set; }
        public string GUID { get; set; }
        public System.DateTime LAST_UPDATED { get; set; }
        public string NAME { get; set; }
        public int CODE { get; set; }
    
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
        public virtual ICollection<RML> RMLS { get; set; }
    }
}
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Collections.Concurrent;
using System.Threading.Tasks;

namespace Tango.CSV
{
    /// <summary>
    /// Represents a component for reading and writing CSV files.
    /// </summary>
    /// <seealso cref="System.IDisposable" />
    public class CsvFile : IDisposable
    {
        protected bool _is_disposed;

        internal protected Stream BaseStream;
        protected static DateTime DateTimeZero = new DateTime();

        /// <summary>
        /// Initializes the <see cref="CsvFile"/> class.
        /// </summary>
        static CsvFile()
        {
            DefaultCsvDefinition = new CsvDefinition
            {
                EndOfLine = "\r\n",
                FieldSeparator = ',',
                TextQualifier = '"'
            };
            UseLambdas = true;
            UseTasks = false;
            FastIndexOfAny = true;
        }

        /// <summary>
        /// Gets or sets the default CSV definition.
        /// </summary>
        public static CsvDefinition DefaultCsvDefinition { get; set; }

        /// <summary>
        /// Gets or sets a value indicating whether [use lambdas].
        /// </summary>
        public static bool UseLambdas { get; set; }

        /// <summary>
        /// Gets or sets a value indicating whether [use tasks].
        /// </summary>
        public static bool UseTasks { get; set; }

        /// <summary>
        /// Gets or sets a value indicating whether [fast index of any].
        /// </summary>
        public static bool FastIndexOfAny { get; set; }

        /// <summary>
        /// Reads the specified CSV source.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="csvSource">The CSV source.</param>
        /// <returns></returns>
        public static IEnumerable<T> Read<T>(CsvSource csvSource) where T : new()
        {
            var csvFileReader = new CsvFileReader<T>(csvSource);
            return (IEnumerable<T>)csvFileReader;
        }

        /// <summary>
        /// Gets the columns.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="csvSource">The CSV source.</param>
        /// <returns></returns>
        public static IEnumerable<string> GetColumns<T>(CsvSource csvSource) where T : new()
        {
            var csvFileReader = new CsvFileReader<T>(csvSource);
            return csvFileReader.Columns;
        }

        /// <summary>
        /// Gets the field separator.
        /// </summary>
        /// <value>
        /// The field separator.
        /// </value>
        public char FieldSeparator { get; private set; }

        /// <summary>
        /// Gets the text qualifier.
        /// </summary>
        /// <value>
        /// The text qualifier.
        /// </value>
        public char TextQualifier { get; private set; }

        /// <summary>
        /// Gets the columns.
        /// </summary>
        /// <value>
        /// The columns.
        /// </value>
        public IEnumerable<String> Columns { get; protected set; }

        /// <summary>
        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
        /// </summary>
        public void Dispose()
        {
            Dispose(true);
        }

        /// <summary>
        /// Releases unmanaged and - optionally - managed resources.
        /// </summary>
        /// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
        protected virtual void Dispose(bool disposing)
        {
            // overriden in derived classes
        }
    }

    /// <summary>
    /// Represents a component for reading and writing CSV files from and to a collection of objects.
    /// </summary>
    /// <typeparam name="T"></typeparam>
    /// <seealso cref="System.IDisposable" />
    public class CsvFile<T> : CsvFile
    {
        private readonly char fieldSeparator;
        private readonly string fieldSeparatorAsString;
        private readonly char[] invalidCharsInFields;
        private readonly StreamWriter streamWriter;
        private readonly char textQualifier;
        private readonly String[] columns;
        private Func<T, object>[] getters;
        readonly bool[] isInvalidCharInFields;
        private int linesToWrite;
        private readonly BlockingCollection<string> csvLinesToWrite = new BlockingCollection<string>(5000);
        private readonly Thread writeCsvLinesTask;
        private Task addAsyncTask;

        /// <summary>
        /// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
        /// </summary>
        /// <param name="csvDestination">The CSV destination.</param>
        public CsvFile(CsvDestination csvDestination)
            : this(csvDestination, null)
        {
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
        /// </summary>
        public CsvFile()
        {
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="CsvFile{T}"/> class.
        /// </summary>
        /// <param name="csvDestination">The CSV destination.</param>
        /// <param name="csvDefinition">The CSV definition.</param>
        public CsvFile(CsvDestination csvDestination, CsvDefinition csvDefinition)
        {
            if (csvDefinition == null)
                csvDefinition = DefaultCsvDefinition;
            this.columns = (csvDefinition.Columns ?? InferColumns(typeof(T))).ToArray();
            this.fieldSeparator = csvDefinition.FieldSeparator;
            this.fieldSeparatorAsString = this.fieldSeparator.ToString(CultureInfo.InvariantCulture);
            this.textQualifier = csvDefinition.TextQualifier;
            this.streamWriter = csvDestination.StreamWriter;

            this.invalidCharsInFields = new[] { '\r', '\n', this.textQualifier, this.fieldSeparator };
            this.isInvalidCharInFields = new bool[256];

            foreach (var c in this.invalidCharsInFields)
            {
                this.isInvalidCharInFields[c] = true;
            }
            this.WriteHeader();

            this.CreateGetters();
            if (CsvFile.UseTasks)
            {
                writeCsvLinesTask = new Thread((o) => this.WriteCsvLines());
                writeCsvLinesTask.Start();
            }
            this.addAsyncTask = Task.Factory.StartNew(() => { });

        }

        /// <summary>
        /// Releases unmanaged and - optionally - managed resources.
        /// </summary>
        /// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                _is_disposed = true;
                // free managed resources
                addAsyncTask.Wait();
                if (csvLinesToWrite != null)
                {
                    csvLinesToWrite.CompleteAdding();
                }
                if (writeCsvLinesTask != null)
                    writeCsvLinesTask.Join();
                this.streamWriter.Close();
            }
        }

        /// <summary>
        /// Infers the columns.
        /// </summary>
        /// <param name="recordType">Type of the record.</param>
        /// <returns></returns>
        protected static IEnumerable<string> InferColumns(Type recordType)
        {
            var columns = recordType
                .GetProperties(BindingFlags.Public | BindingFlags.Instance)
                .Where(pi => pi.GetIndexParameters().Length == 0
                    && pi.GetSetMethod() != null
                    && !Attribute.IsDefined(pi, typeof(CsvIgnoreAttribute)))
                .Select(pi => pi.Name)
                .Concat(recordType
                    .GetFields(BindingFlags.Public | BindingFlags.Instance)
                    .Where(fi => !Attribute.IsDefined(fi, typeof(CsvIgnoreAttribute)))
                    .Select(fi => fi.Name))
                .ToList();
            return columns;
        }

        /// <summary>
        /// Writes the CSV lines.
        /// </summary>
        private void WriteCsvLines()
        {
            int written = 0;
            foreach (var csvLine in csvLinesToWrite.GetConsumingEnumerable())
            {
                this.streamWriter.WriteLine(csvLine);
                written++;
            }
            Interlocked.Add(ref this.linesToWrite, -written);
        }

        /// <summary>
        /// Appends the specified record.
        /// </summary>
        /// <param name="record">The record.</param>
        public void Append(T record)
        {
            if (_is_disposed)
            {
                return;
            }

            if (CsvFile.UseTasks)
            {

                var linesWaiting = Interlocked.Increment(ref this.linesToWrite);
                Action<Task> addRecord = (t) =>
                {
                    if (!_is_disposed)
                    {
                        var csvLine = this.ToCsv(record);
                        this.csvLinesToWrite.Add(csvLine);
                    }
                };

                if (linesWaiting < 10000)
                    this.addAsyncTask = this.addAsyncTask.ContinueWith(addRecord);
                else
                    addRecord(null);
            }
            else
            {
                var csvLine = this.ToCsv(record);
                this.streamWriter.WriteLine(csvLine);
            }
        }

        /// <summary>
        /// Finds the getter.
        /// </summary>
        /// <param name="c">The c.</param>
        /// <param name="staticMember">if set to <c>true</c> [static member].</param>
        /// <returns></returns>
        private static Func<T, object> FindGetter(string c, int index, bool staticMember)
        {
            var flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.IgnoreCase | (staticMember ? BindingFlags.Static : BindingFlags.Instance);
            Func<T, object> func = null;

            PropertyInfo pi = null;
            pi = typeof(T).GetProperty(c);

            if (pi == null) //Then try get by column index,
            {
                pi = typeof(T).GetProperties()[index];
                pi = typeof(T).GetProperties()[index]; //Workaround for some weired problem that makes the program not take the value.
            }

            if (CsvFile.UseLambdas)
            {
                Expression expr = null;
                ParameterExpression parameter = Expression.Parameter(typeof(T), "r");
                Type type = null;

                if (pi != null)
                {
                    type = pi.PropertyType;
                    expr = Expression.Property(parameter, pi.Name);
                }
                if (expr != null)
                {
                    Expression<Func<T, object>> lambda;
                    if (type.IsValueType)
                    {
                        lambda = Expression.Lambda<Func<T, object>>(Expression.TypeAs(expr, typeof(object)), parameter);
                    }
                    else
                    {
                        lambda = Expression.Lambda<Func<T, object>>(expr, parameter);
                    }
                    func = lambda.Compile();
                }
            }
            else
            {
                if (pi != null)
                    func = o => pi.GetValue(o, null);
            }
            return func;
        }

        /// <summary>
        /// Creates the getters.
        /// </summary>
        private void CreateGetters()
        {
            var list = new List<Func<T, object>>();

            for (int i = 0; i < columns.Length; i++)
            {
                Func<T, Object> func = null;
                var propertyName = (columns[i].IndexOf(' ') < 0 ? columns[i] : columns[i].Replace(" ", ""));
                func = FindGetter(columns[i], i, false) ?? FindGetter(columns[i], i, true);

                list.Add(func);
            }
            this.getters = list.ToArray();
        }

        /// <summary>
        /// To the CSV.
        /// </summary>
        /// <param name="record">The record.</param>
        /// <returns></returns>
        /// <exception cref="System.ArgumentException">Cannot be null;record</exception>
        private string ToCsv(T record)
        {
            if (record == null)
                throw new ArgumentException("Cannot be null", "record");

            string[] csvStrings = new string[getters.Length];

            for (int i = 0; i < getters.Length; i++)
            {
                var getter = getters[i];
                object fieldValue = getter == null ? null : getter(record);
                csvStrings[i] = this.ToCsvString(fieldValue);
            }
            return string.Join(this.fieldSeparatorAsString, csvStrings);

        }

        /// <summary>
        /// To the CSV string.
        /// </summary>
        /// <param name="o">The o.</param>
        /// <returns></returns>
        private string ToCsvString(object o)
        {
            if (o != null)
            {
                string valueString = o as string ?? Convert.ToString(o, CultureInfo.CurrentUICulture);
                if (RequireQuotes(valueString))
                {
                    var csvLine = new StringBuilder();
                    csvLine.Append(this.textQualifier);
                    foreach (char c in valueString)
                    {
                        if (c == this.textQualifier)
                            csvLine.Append(c); // double the double quotes
                        csvLine.Append(c);
                    }
                    csvLine.Append(this.textQualifier);
                    return csvLine.ToString();
                }
                else
                    return valueString;
            }
            return string.Empty;
        }

        /// <summary>
        /// Requires the quotes.
        /// </summary>
        /// <param name="valueString">The value string.</param>
        /// <returns></returns>
        private bool RequireQuotes(string valueString)
        {
            if (CsvFile.FastIndexOfAny)
            {
                var len = valueString.Length;
                for (int i = 0; i < len; i++)
                {
                    char c = valueString[i];
                    if (c <= 255 && this.isInvalidCharInFields[c])
                        return true;
                }
                return false;
            }
            else
            {
                return valueString.IndexOfAny(this.invalidCharsInFields) >= 0;
            }
        }

        /// <summary>
        /// Writes the header.
        /// </summary>
        private void WriteHeader()
        {
            var csvLine = new StringBuilder();
            for (int i = 0; i < this.columns.Length; i++)
            {
                if (i > 0)
                    csvLine.Append(this.fieldSeparator);
                csvLine.Append(this.ToCsvString(this.columns[i]));
            }
            this.streamWriter.WriteLine(csvLine.ToString());
        }
    }
}