Reading XML Streams (Reading large files)

In This Topic

When deserializing XML data using Liquid XML Objects, the entire XML document is loaded into the generated Liquid XML Object Model. For most applications this is fine as the XML documents are of a manageable size, however if you are working on large XML files then loading the whole document may not be possible, the following demonstrates how to read a large XML document in management chunks.

Simple Log File Example

This example will show how to read the log entries from the file one at a time, allowing extremely large files to be processed with minimal resources.

Sample XML Data	Copy Code
<?xml version="1.0" encoding="utf-8"?> <!-- Created with Liquid Studio (https://www.liquid-technologies.com) --> <LogFile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="SimpleLogFile.xsd"> <LogEntry> <Date>1992-11-18T20:07:25.64</Date> <Severity>Error</Severity> <Message>The Widget could not be found</Message> </LogEntry> <LogEntry> <Date>2004-09-06T05:53:58.85</Date> <Severity>Warning</Severity> <Message>Invalid Path</Message> </LogEntry> <LogEntry> <Date>2007-08-24T13:55:53.07</Date> <Severity>Warning</Severity> <Message>Could not find the correct thing</Message> </LogEntry> <LogEntry> <Date>1976-09-05T09:18:22.00</Date> <Severity>Info</Severity> <Message>Disk space low</Message> </LogEntry> </LogFile>

Sample XML Data

Copy Code

<?xml version="1.0" encoding="utf-8"?>
<!-- Created with Liquid Studio (https://www.liquid-technologies.com) -->
<LogFile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="SimpleLogFile.xsd">
   <LogEntry>
      <Date>1992-11-18T20:07:25.64</Date>
      <Severity>Error</Severity>
      <Message>The Widget could not be found</Message>
   </LogEntry>
   <LogEntry>
      <Date>2004-09-06T05:53:58.85</Date>
      <Severity>Warning</Severity>
      <Message>Invalid Path</Message>
   </LogEntry>
   <LogEntry>
      <Date>2007-08-24T13:55:53.07</Date>
      <Severity>Warning</Severity>
      <Message>Could not find the correct thing</Message>
   </LogEntry>
   <LogEntry>
      <Date>1976-09-05T09:18:22.00</Date>
      <Severity>Info</Severity>
      <Message>Disk space low</Message>
   </LogEntry>
</LogFile>

Generated Code	Copy Code
#region Elements [LxSimpleElementDefinition("LogFile", "", ElementScopeType.GlobalElement)] public partial class LogFileElm : LiquidXmlObjects.SimpleLogFile.LxBase { [LxElementCt("LogEntry", "", MinOccurs = 0, MaxOccurs = LxConstants.Unbounded)] public List<LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt> LogEntries { get; } = new List<LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt>(); } #endregion #region Complex Types [LxSimpleComplexTypeDefinition("LogEntryType", "")] public partial class LogEntryTypeCt : LiquidXmlObjects.SimpleLogFile.LxBase { [LxElementValue("Date", "", LxValueType.Value, XsdType.XsdDateTime, MinOccurs = 1, MaxOccurs = 1)] public LiquidTechnologies.XmlObjects.LxDateTime Date { get; set; } [LxElementValue("Severity", "", LxValueType.Enum, XsdType.Enum, MinOccurs = 1, MaxOccurs = 1, WhiteSpace = WhiteSpaceType.Preserve)] public LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt.SeverityEnum Severity { get; set; } [LxElementValue("Message", "", LxValueType.Value, XsdType.XsdString, MinOccurs = 1, MaxOccurs = 1)] public System.String Message { get; set; } = ""; public enum SeverityEnum { [LxEnumValue("Error")] Error, [LxEnumValue("Warning")] Warning, [LxEnumValue("Info")] Info, } } #endregion

Generated Code

Copy Code

    #region Elements
    [LxSimpleElementDefinition("LogFile", "", ElementScopeType.GlobalElement)]
    public partial class LogFileElm : LiquidXmlObjects.SimpleLogFile.LxBase
    {
        [LxElementCt("LogEntry", "", MinOccurs = 0, MaxOccurs = LxConstants.Unbounded)]
        public List<LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt> LogEntries { get; } = new List<LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt>();
    }
    #endregion

    #region Complex Types
    [LxSimpleComplexTypeDefinition("LogEntryType", "")]
    public partial class LogEntryTypeCt : LiquidXmlObjects.SimpleLogFile.LxBase
    {
        [LxElementValue("Date", "", LxValueType.Value, XsdType.XsdDateTime, MinOccurs = 1, MaxOccurs = 1)]
        public LiquidTechnologies.XmlObjects.LxDateTime Date { get; set; }
        [LxElementValue("Severity", "", LxValueType.Enum, XsdType.Enum, MinOccurs = 1, MaxOccurs = 1, WhiteSpace = WhiteSpaceType.Preserve)]
        public LiquidXmlObjects.SimpleLogFile.Ns.LogEntryTypeCt.SeverityEnum Severity { get; set; }
        [LxElementValue("Message", "", LxValueType.Value, XsdType.XsdString, MinOccurs = 1, MaxOccurs = 1)]
        public System.String Message { get; set; } = "";
        public enum SeverityEnum
        {
            [LxEnumValue("Error")]   Error,
            [LxEnumValue("Warning")] Warning,
            [LxEnumValue("Info")]    Info,
        }
    }
    #endregion

Sample Code	Copy Code
LxSerializer<LogEntryTypeCt> logEntrySerializer = new LxSerializer<LogEntryTypeCt>(); using (XmlReader xmlReader = new XmlTextReader(@"SimpleLogFile_SampleFile.xml")) { if (xmlReader.ReadToDescendant("LogFile") == false) throw new InvalidOperationException("Expecting the root element to be 'LogFile'"); while ((xmlReader.Name == "LogEntry") \|\| // already at a LogEntry element, so just process it xmlReader.ReadToFollowing("LogEntry")) // read until we get to a LogEntry element { var logEntry = logEntrySerializer.DeserializeSnippet(xmlReader); Console.WriteLine($"Processing LogEntry : {logEntry.Date} ({logEntry.Severity}) - {logEntry.Message}"); } }

Sample Code

Copy Code

LxSerializer<LogEntryTypeCt> logEntrySerializer = new LxSerializer<LogEntryTypeCt>();

using (XmlReader xmlReader = new XmlTextReader(@"SimpleLogFile_SampleFile.xml"))
{
    if (xmlReader.ReadToDescendant("LogFile") == false)
        throw new InvalidOperationException("Expecting the root element to be 'LogFile'");

    while ((xmlReader.Name == "LogEntry") ||        // already at a LogEntry element, so just process it
            xmlReader.ReadToFollowing("LogEntry"))  // read until we get to a LogEntry element
    {
        var logEntry = logEntrySerializer.DeserializeSnippet(xmlReader);
        Console.WriteLine($"Processing LogEntry : {logEntry.Date} ({logEntry.Severity}) - {logEntry.Message}");
    }
}

The sample code above uses an XmlReader to open the XML document, and navigates through the XML document until it hits the element containing the child items to process (typically large files have 1 parent element containing 1000s of child records). Then for each child element (LogEntry) it deserializes it into a LogEntryTypeCt (which is the C# class generated to handle the xs:complexType LogEntryType).

Its not uncommon for large XML files (especially log files) not to be terminated correctly (i.e. they are missing the closing element tag), the approach shown here allows this error to be ignored.

Sample Output	Copy Code
Processing LogEntry : 1992-11-18T20:07:25.64 (Error) - The Widget could not be found Processing LogEntry : 2004-09-06T05:53:58.85 (Warning) - Invalid Path Processing LogEntry : 2007-08-24T13:55:53.07 (Warning) - Could not find the correct thing Processing LogEntry : 1976-09-05T09:18:22 (Info) - Disk space low

Sample Output

Copy Code

Processing LogEntry : 1992-11-18T20:07:25.64 (Error) - The Widget could not be found
Processing LogEntry : 2004-09-06T05:53:58.85 (Warning) - Invalid Path
Processing LogEntry : 2007-08-24T13:55:53.07 (Warning) - Could not find the correct thing
Processing LogEntry : 1976-09-05T09:18:22 (Info) - Disk space low

Advanced Cases

If the XML snippet to be deserialized contains an xsi:type, then the correct C# object will be created for it, so if we have the following schema, the LogEntry could be represented by a LogEntryTypeCt or an EnhancedLogEntryTypeCt.

Advanced	Copy Code
<?xml version="1.0" encoding="utf-8"?> <!-- Created with Liquid Studio (https://www.liquid-technologies.com) --> <LogFile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="AdvancedLogFile.xsd"> <LogEntry xsi:type="EnhancedLogEntryType"> <Date>1992-11-18T20:07:25.64</Date> <Severity>Error</Severity> <Message>The Widget could not be found</Message> <Suggestion>Look for the widget</Suggestion> </LogEntry> <LogEntry> <Date>2004-09-06T05:53:58.85</Date> <Severity>Warning</Severity> <Message>Invalid Path</Message> </LogEntry> <LogEntry> <Date>2007-08-24T13:55:53.07</Date> <Severity>Warning</Severity> <Message>Could not find the correct thing</Message> </LogEntry> <LogEntry xsi:type="EnhancedLogEntryType"> <Date>1976-09-05T09:18:22.00</Date> <Severity>Info</Severity> <Message>Disk space low</Message> <Suggestion>Add more disk space</Suggestion> </LogEntry> </LogFile>

Advanced

Copy Code

<?xml version="1.0" encoding="utf-8"?>
<!-- Created with Liquid Studio (https://www.liquid-technologies.com) -->
<LogFile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="AdvancedLogFile.xsd">
   <LogEntry xsi:type="EnhancedLogEntryType">
      <Date>1992-11-18T20:07:25.64</Date>
      <Severity>Error</Severity>
      <Message>The Widget could not be found</Message>
      <Suggestion>Look for the widget</Suggestion>
   </LogEntry>
   <LogEntry>
      <Date>2004-09-06T05:53:58.85</Date>
      <Severity>Warning</Severity>
      <Message>Invalid Path</Message>
   </LogEntry>
   <LogEntry>
      <Date>2007-08-24T13:55:53.07</Date>
      <Severity>Warning</Severity>
      <Message>Could not find the correct thing</Message>
   </LogEntry>
   <LogEntry xsi:type="EnhancedLogEntryType">
      <Date>1976-09-05T09:18:22.00</Date>
      <Severity>Info</Severity>
      <Message>Disk space low</Message>
      <Suggestion>Add more disk space</Suggestion>
   </LogEntry>
</LogFile>

Sample Code	Copy Code
LxSerializer<LogEntryTypeCt> logEntrySerializer = new LxSerializer<LogEntryTypeCt>(); using (XmlReader xmlReader = new XmlTextReader(@"AdvancedLogFile_SampleFile.xml")) { if (xmlReader.ReadToDescendant("LogFile") == false) throw new InvalidOperationException("Expecting the root element to be 'LogFile'"); while ((xmlReader.Name == "LogEntry") \|\| // already at a LogEntry element, so just process it xmlReader.ReadToFollowing("LogEntry")) // read until we get to a LogEntry element { var logEntry = logEntrySerializer.DeserializeSnippet(xmlReader); if (logEntry is EnhancedLogEntryTypeCt enhancedLogEntry) Console.WriteLine($"Processing LogEntry : {enhancedLogEntry.Date} ({enhancedLogEntry.Severity}) - {enhancedLogEntry.Message} [Try this : {enhancedLogEntry.Suggestion}]"); else Console.WriteLine($"Processing LogEntry : {logEntry.Date} ({logEntry.Severity}) - {logEntry.Message}"); } }

Sample Code

Copy Code

LxSerializer<LogEntryTypeCt> logEntrySerializer = new LxSerializer<LogEntryTypeCt>();

using (XmlReader xmlReader = new XmlTextReader(@"AdvancedLogFile_SampleFile.xml"))
{
    if (xmlReader.ReadToDescendant("LogFile") == false)
        throw new InvalidOperationException("Expecting the root element to be 'LogFile'");

    while ((xmlReader.Name == "LogEntry") ||        // already at a LogEntry element, so just process it
            xmlReader.ReadToFollowing("LogEntry"))  // read until we get to a LogEntry element
    {
        var logEntry = logEntrySerializer.DeserializeSnippet(xmlReader);
        if (logEntry is EnhancedLogEntryTypeCt enhancedLogEntry)
            Console.WriteLine($"Processing LogEntry : {enhancedLogEntry.Date} ({enhancedLogEntry.Severity}) - {enhancedLogEntry.Message} [Try this : {enhancedLogEntry.Suggestion}]");
        else
            Console.WriteLine($"Processing LogEntry : {logEntry.Date} ({logEntry.Severity}) - {logEntry.Message}");
    }
}

Sample Output	Copy Code
Processing LogEntry : 1992-11-18T20:07:25.64 (Error) - The Widget could not be found [Try this : Look for the widget] Processing LogEntry : 2004-09-06T05:53:58.85 (Warning) - Invalid Path Processing LogEntry : 2007-08-24T13:55:53.07 (Warning) - Could not find the correct thing Processing LogEntry : 1976-09-05T09:18:22 (Info) - Disk space low [Try this : Add more disk space]

Sample Output

Copy Code

Processing LogEntry : 1992-11-18T20:07:25.64 (Error) - The Widget could not be found [Try this : Look for the widget]
Processing LogEntry : 2004-09-06T05:53:58.85 (Warning) - Invalid Path
Processing LogEntry : 2007-08-24T13:55:53.07 (Warning) - Could not find the correct thing
Processing LogEntry : 1976-09-05T09:18:22 (Info) - Disk space low [Try this : Add more disk space]

Reference

DeserializeSnippet Method

Send Feedback