After being saved, the Rubber Stamp objects load as Stamp objects. This is probably why you need to create the stamp object when loading the saved files.
I used the same project that you attached above, and I added the following lines after theManager.Objects.Add(rubberstampObj)
'-------------------------------
Dim stampObj As New AnnAutomationObject
stampObj.Id = AnnAutomationManager.RubberStampObjectId
stampObj.Name = "Stamp"
Dim stam As New AnnStampObject
rubstam.Pen = New AnnPen(Color.Black, New AnnLength(1, AnnUnit.Display))
rubstam.Brush = New AnnSolidBrush(Color.Yellow)
rubstam.Font = New AnnFont("Arial", New AnnLength(48, AnnUnit.Point), FontStyle.Regular)
stampObj.Object = stam
stampObj.DrawDesignerType = GetType(AnnTextDrawDesigner)
stampObj.EditDesignerType = GetType(AnnRectangleEditDesigner)
stampObj.RunDesignerType = GetType(AnnRunDesigner)
g = Graphics.FromImage(btmp)
g.FillRectangle(Brushes.DarkMagenta, 0, 0, 16, 16)
stampObj.ToolBarImage = Nothing
stampObj.ToolBarToolTipText = "Draw new stamp object"
stampObj.DrawCursor = Cursors.Cross
stampObj.ContextMenu = AnnAutomationManager.CreateDefaultObjectContextMenu(stampObj.Id)
stampObj.UseRotateControlPoints = True
manager.Objects.Add(stampObj)
'-------------------------------