C# Basic Stamp Tokenizer Class

Toft

Here's what I've got so far;

But I'm having trouble making calls to the TestAlign functions and Compile Functions. I think I'm not marshalling properly?

I try:

        IntPtr RecPtr = IntPtr.Zero;
        IntPtr TokPtr = IntPtr.Zero;
        Stamp.TModuleRec myRec = new Stamp.TModuleRec();
        RecPtr = Marshal.AllocCoTaskMem(4);
        Marshal.StructureToPtr(myRec, RecPtr, true);
        Stamp.tokenizer_test_align(ref RecPtr);

And it just fails. Anyone have ideas?

---

SharpStamp.cs

---

using System;
using System.Collections.Generic;
using System.Text;
using System.Runtime.InteropServices;

namespace CSStampServer
{
    class Stamp
    {
        /// <summary>
        /// The value returned is in the format: XYY ;where X is the major version number and Y is the minor. 
        /// For example, if the Version function returned 116, that indicates the version of the tokenizer is 1.16.
        /// If Version returned 123, that would indicate the tokenizer is version 1.23.
        /// </summary>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.StdCall, EntryPoint = "Version")]
        public static extern int tokenizer_version();

        /// <summary>
        /// Tests the alignment and sets values to, 0, 1, 2, 3, ... 32.
        /// </summary>
        /// <param name="Rec"></param>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.StdCall, EntryPoint = "TestRecAlignment")]
        public static extern bool tokenizer_test_align(ref IntPtr /* TModuleRec* */ Rec);

        /// <summary>
        /// /// The Compile function for the basic stamp tokenizer.
        /// </summary>
        /// <param name="Rec">A pointer to an existing TModuleRec structure.</param>
        /// <param name="Src">A pointer to an existing Source buffer.</param>
        /// <param name="DirectivesOnly">A Boolean value that provides an option of only tokenizing the &#8220;compiler directives&#8221; from the source code, rather than the entire source.
        /// This option is helpful when the calling program needs to determine only the target module, serial port or project files that may be specified by the PBASIC source code.
        /// TRUE : Causes the tokenizer to only parse the directives from the source code.
        /// FALSE : (Normal mode) Causes the tokenizer to parse the entire source code, including all directives (depending on the ParseStampDirective parameter).</param>
        /// <param name="ParseStampDirective">A Boolean value that provides an option of parsing the Stamp directive from the source code, rather than accepting a value in the TargetModule field of the TModuleRec structure.
        /// TRUE : (Normal mode) Causes the tokenizer to parse the Stamp directive, if present, from the source code.
        /// If a valid directive is found, the compiler stores the value of the target module in the TargetModule field and compiles. If no Stamp directive is found, the tokenizer generates an error.
        /// FALSE : Causes the tokenizer to ignore any Stamp directives in the source and instead use the value currently in the TargetModule field to compile the code.
        /// This means, the TargetModule field MUST be set to the proper value BEFORE calling Compile.</param>
        /// <param name="Ref">An optional pointer to an existing TSrcTokReference buffer. Set to NULL when not used.</param>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.StdCall, EntryPoint = "Compile")]
        public static extern bool tokenizer_compile(IntPtr /* TModuleRec* */ Rec,
            [noparse][[/noparse]MarshalAs(UnmanagedType.LPStr)]IntPtr /* char* */ Src,
            [noparse][[/noparse]MarshalAs(UnmanagedType.Bool)]bool DirectivesOnly,
            [noparse][[/noparse]MarshalAs(UnmanagedType.Bool)]bool ParseStampDirective,
            IntPtr /* TSrcTokReference* */ Ref);

        #region Unmanaged Structs Implementation

        [noparse][[/noparse]StructLayout(LayoutKind.Sequential)]
        internal struct STAMP_IF
        {
            public IntPtr /* TSrcTokReference* */   SrcTokenReference;
            public string Name;
            public string Decription;
            public IntPtr /* TModuleRec* */         TokenModule;
        }

        /// <summary>
        /// This structure is optional since it is intended for use by PBASIC simulators.
        /// </summary>
        [noparse][[/noparse]StructLayout(LayoutKind.Sequential)]
        internal struct TSrkTokReference
        {
            [noparse][[/noparse]MarshalAs(UnmanagedType.U2)]
            public ushort SrcStart;
            [noparse][[/noparse]MarshalAs(UnmanagedType.U2)]
            public ushort TokStart;
        }

        /// <summary>
        /// This is the structure that is used for the Basic Stamp Tokenization process. (The information sent to the Compiler.)
        /// </summary>
        [noparse][[/noparse]StructLayout(LayoutKind.Sequential)]
        internal class TModuleRec
        {
            public bool Succeeded;
            public string Error;
            public bool DebugFlag;

            public byte TargetModule;
            public int TargetStart;

            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 7)]
            public IntPtr[noparse][[/noparse]] /* char** */ ProjectFiles;
            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 7)]
            public IntPtr[noparse][[/noparse]] /* int* */ ProjectFilesStart;

            public string Port;
            public int PortStart;

            public int LanguageVersion;
            public int LanguageStart;

            public int SourceSize;
            public int ErrorStart;
            public int ErrorLength;

            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 2048)]
            public byte[noparse][[/noparse]] EEPROM;
            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 2048)]
            public byte[noparse][[/noparse]] EEPROMFlags;

            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
            public byte[noparse][[/noparse]] VarCounts;
            public byte PacketCount;
            [noparse][[/noparse]MarshalAs(UnmanagedType.ByValArray, SizeConst = 2304)]
            public byte[noparse][[/noparse]] PacketBuffer;            
        }

        #endregion

        public static string Version
        {
            get
            {
                try
                {
                    return tokenizer_version().ToString();
                }
                catch
                {
                    return "Parallax Tokenizer version can't be identified or the tokenizer.dll is not installed.";
                }
            }
        }
    }
}

Adrian Schneider

I am not really familiar with C#, so I may miss the point. However, the problem might be that you do not marshall every field in your
TModuleRec and it thus gets missaligned. Fill your TModuleRec structure with characteristic values and check the memory layout with the
debugger or a hexdump.
Hope it helps.
Adrian

Toft

It actually had nothing to do without calling the marshaller for the other options, when passing these references to the TestAlign function it ended up working perfeclt well with unsafe pointers.

Here's my example:

public unsafe void TestRecAlign() {
            //Verify Stamp Record Alignment (TestAlignRec)
            fixed (Stamp.TModuleRec* p = &myRec)
            {
                if (Stamp.tokenizer_test_align(p) == true)
                {
                    int i = 0;

                    richTextBox1.Text = "Test align returned true" + Environment.NewLine + Environment.NewLine;
                    richTextBox2.Text = "Values are..." + Environment.NewLine;

                    richTextBox2.Text += "Succeeded: " + p->Succeeded.ToString() + Environment.NewLine;
                    richTextBox2.Text += "Error: " + p->Error.ToString() + Environment.NewLine;
                    richTextBox2.Text += "DebugFlag: " + p->DebugFlag.ToString() + Environment.NewLine;

                    richTextBox2.Text += "TargetModule: " + p->TargetModule.ToString() + Environment.NewLine;
                    richTextBox2.Text += "TargetStart: " + p->TargetStart.ToString() + Environment.NewLine;

                    for (i = 0; i < 6; i++)
                    {
                        richTextBox2.Text += "ProjectFiles[noparse][[/noparse]" + i.ToString() + "]: " + Marshal.PtrToStringAnsi((IntPtr)p->ProjectFiles[i]) + Environment.NewLine;
                    }

                    for (i = 0; i < 6; i++)
                    {
                        richTextBox2.Text += "ProjectFilesStart[noparse][[/noparse]" + i.ToString() + "]: " + p->ProjectFilesStart[i].ToString() + Environment.NewLine;
                    }

                    richTextBox2.Text += "Port: " + Marshal.PtrToStringAnsi(p->Port) + Environment.NewLine;
                    richTextBox2.Text += "PortStart: " + p->PortStart.ToString() + Environment.NewLine;
                    richTextBox2.Text += "LanguageVersion: " + p->LanguageVersion.ToString() + Environment.NewLine;
                    richTextBox2.Text += "LanguageStart: " + p->LanguageStart.ToString() + Environment.NewLine;
                    richTextBox2.Text += "SourceSize: " + p->SourceSize.ToString() + Environment.NewLine;
                    richTextBox2.Text += "ErrorStart: " + p->ErrorStart.ToString() + Environment.NewLine;
                    richTextBox2.Text += "ErrorLength: " + p->ErrorLength.ToString() + Environment.NewLine;

                    richTextBox2.Text += "EEPROM: ";
                    for (i = 0; i < 2048; i++)
                    {
                        richTextBox2.Text += p->EEPROM[i].ToString();
                    }
                    richTextBox2.Text += Environment.NewLine;

                    richTextBox2.Text += "EEPROMFlags: ";
                    for (i = 0; i < 2048; i++)
                    {
                        richTextBox2.Text += p->EEPROMFlags[i].ToString();
                    }
                    richTextBox2.Text += Environment.NewLine;

                    for (i = 0; i < 3; i++)
                    {
                        richTextBox2.Text += "VarCounts[noparse][[/noparse]" + i.ToString() + "]: " + p->VarCounts[i].ToString() + Environment.NewLine;
                    }

                    richTextBox2.Text += "PacketCount: " + p->PacketCount.ToString() + Environment.NewLine;

                    richTextBox2.Text += "PacketBuffer: ";
                    for (i = 0; i < 2304; i++)
                    {
                        richTextBox2.Text += p->PacketBuffer[i].ToString();
                    }
                    richTextBox2.Text += Environment.NewLine;

                    return true;
                }
                else
                {
                    return false;
                }
            }

        }[/i][/i][/i][/i][/i][/i]

For -calling- the rec align function, the actual struct (and all of the background stuff), is declared as follows;

class Stamp
    {
        /// <summary>
        /// The value returned is in the format: XYY ;where X is the major version number and Y is the minor. 
        /// For example, if the Version function returned 116, that indicates the version of the tokenizer is 1.16.
        /// If Version returned 123, that would indicate the tokenizer is version 1.23.
        /// </summary>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi,
            CallingConvention = CallingConvention.StdCall, EntryPoint = "Version")]
        public static extern int tokenizer_version();

        /// <summary>
        /// Tests the alignment and sets values to, 0, 1, 2, 3, ... 32.
        /// </summary>
        /// <param name="Rec"></param>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi,
            CallingConvention = CallingConvention.StdCall, EntryPoint = "TestRecAlignment")]
        public unsafe static extern bool tokenizer_test_align(TModuleRec* /* TModuleRec* */ Rec);

        /// <summary>
        /// /// The Compile function for the basic stamp tokenizer.
        /// </summary>
        /// <param name="Rec">A pointer to an existing TModuleRec structure.</param>
        /// <param name="Src">A pointer to an existing Source buffer.</param>
        /// <param name="DirectivesOnly">A Boolean value that provides an option of only tokenizing the &#8220;compiler directives&#8221;
        /// from the source code, rather than the entire source.
        /// This option is helpful when the calling program needs to determine only the target module, serial port or
        /// project files that may be specified by the PBASIC source code.
        /// TRUE : Causes the tokenizer to only parse the directives from the source code.
        /// FALSE : (Normal mode) Causes the tokenizer to parse the entire source code, including all directives
        /// (depending on the ParseStampDirective parameter).</param>
        /// <param name="ParseStampDirective">A Boolean value that provides an option of parsing the Stamp directive
        /// from the source code, rather than accepting a value in the TargetModule field of the TModuleRec structure.
        /// TRUE : (Normal mode) Causes the tokenizer to parse the Stamp directive, if present, from the source code.
        /// If a valid directive is found, the compiler stores the value of the target module in the TargetModule
        /// field and compiles. If no Stamp directive is found, the tokenizer generates an error.
        /// FALSE : Causes the tokenizer to ignore any Stamp directives in the source and instead use the value
        /// currently in the TargetModule field to compile the code.
        /// This means, the TargetModule field MUST be set to the proper value BEFORE calling Compile.</param>
        /// <param name="Ref">An optional pointer to an existing TSrcTokReference buffer. Set to NULL when not used.</param>
        /// <returns></returns>
        [noparse][[/noparse]DllImport("tokenizer.dll", CharSet = CharSet.Ansi,
            CallingConvention = CallingConvention.StdCall, EntryPoint = "Compile")]
        public unsafe static extern bool tokenizer_compile(TModuleRec* /* TModuleRec* */ Rec,
            [noparse][[/noparse]MarshalAs(UnmanagedType.LPStr)] string Src, bool DirectivesOnly,
            bool ParseStampDirective, TSrkTokReference* /* TSrcTokReference* */ Ref);

        #region Unmanaged Structs Implementation

        [noparse][[/noparse]StructLayout(LayoutKind.Sequential)]
        internal struct STAMP_IF
        {
            public IntPtr /* TSrcTokReference* */   SrcTokenReference;
            public string                           Name;
            public string                           Decription;
            public IntPtr /* TModuleRec* */         TokenModule;
            public SerialPort                       serialPort;
            public int   /* Port for remoting */    remotePort;
        }

        /// <summary>
        /// This structure is optional since it is intended for use by PBASIC simulators.
        /// </summary>
        [noparse][[/noparse]StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
            unsafe internal struct TSrkTokReference
        {
            public ushort SrcStart;
            public ushort TokStart;
        }

        /// <summary>
        /// This is the structure that is used for the Basic Stamp Tokenization process.
        /// (The information sent to the Compiler.)
        /// </summary>
        [noparse][[/noparse]StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack = 4)]
            unsafe internal struct TModuleRec
        {
            public byte Succeeded; /*Is a bool*/
            public IntPtr Error; /* Marshal to AnsiString */
            public byte DebugFlag; /*Is a bool*/

            public byte TargetModule;
            public int TargetStart;

            public fixed int ProjectFiles[noparse][[/noparse]7]; /* Cast To IntPtr */
            public fixed int ProjectFilesStart[noparse][[/noparse]7];
            
            public IntPtr Port; /* Marshal to AnsiString */
            public int PortStart;

            public int LanguageVersion;
            public int LanguageStart;

            public int SourceSize;
            public int ErrorStart;
            public int ErrorLength;

            public fixed byte EEPROM[noparse][[/noparse]2048];
            public fixed byte EEPROMFlags[noparse][[/noparse]2048];

            public fixed byte VarCounts;
            public byte PacketCount;
            public fixed byte PacketBuffer[noparse][[/noparse]2304];
        }

        #endregion

        public static string Version
        {
            get
            {
                try
                {
                    return tokenizer_version().ToString();
                }
                catch
                {
                    return "Parallax Tokenizer version can't be identified or the tokenizer.dll is not installed.";
                }
            }
        }
    }

I'll try to get this actually wrapped into an entire library with collections so that if other people want to use the tokenizer in C# they can!