by Mark Bowytz in CodeSOD on Edit

Besides contributing at @TheDailyWTF, I write DevDisasters for Visual Studio Magazine, and involved in various side projects including child rearing and marriage.

If you ever hit upon a scenario where you need to mine meaningful text data out of any set of HTML files, you will likely find yourself facing a potentially hairy situation.

With the ads, silly social media add-ons, sidebars, toolbars, and likely WTF-level web page coding practices, unless you’re looking at a set of pure vanilla, consistently designed pages, it can be a big mess.

Your average developer, when faced with this situation, would do what any sane person would do – first try to tackle the problem on their own and after a few frustrating iterations, eventually turn to the Internet to solve their problem. Thankfully, as it turns out, the problem of parsing text out of web code has been solved several times over effectively turning your development task into an integration task. Hooray!

The expert, above average coders take a different route – they persevere. They will hack away at the problem and end up with a monster function like the one that Joe discovered. According to him, he caught it in the act after noticing that (only) 700k files were processing very slowly one morning.

public static string StripTagsCharArray ( string source ) { try { string result ; // Remove HTML Development formatting // Replace line breaks with space // because browsers inserts space result = source . Replace ( " \r " , " " ) ; // Replace line breaks with space // because browsers inserts space result = result . Replace ( "

" , " " ) ; // Remove step-formatting result = result . Replace ( " \t " , string . Empty ) ; // Remove repeating spaces because browsers ignore them result = System . Text . RegularExpressions . Regex . Replace ( result , @ " ( )+ " , " " ) ; // Remove the header (prepare first by clearing attributes) result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*head([^>])*> " , " <head> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " (<( )*(/)( )*head( )*>) " , " </head> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , " (<head>).*(</head>) " , string . Empty , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // remove all scripts (prepare first by clearing attributes) result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*script([^>])*> " , " <script> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " (<( )*(/)( )*script( )*>) " , " </script> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; //result = System.Text.RegularExpressions.Regex.Replace(result, // @"(<script>)([^(<script>\.</script>)])*(</script>)", // string.Empty, // System.Text.RegularExpressions.RegexOptions.IgnoreCase); result = System . Text . RegularExpressions . Regex . Replace ( result , @ " (<script>).*(</script>) " , string . Empty , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // remove all styles (prepare first by clearing attributes) result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*style([^>])*> " , " <style> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " (<( )*(/)( )*style( )*>) " , " </style> " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , " (<style>).*(</style>) " , string . Empty , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // insert tabs in spaces of <td> tags result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*td([^>])*> " , " \t " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // insert line breaks in places of <BR> and <LI> tags result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*br( )*> " , " \r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*li( )*> " , " \r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // insert line paragraphs (double line breaks) in place // if <P>, <DIV> and <TR> tags result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*div([^>])*> " , " \r\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*tr([^>])*> " , " \r\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <( )*p([^>])*> " , " \r\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // Remove remaining tags like <a>, links, images, // comments etc - anything that's enclosed inside < > result = System . Text . RegularExpressions . Regex . Replace ( result , @ " <[^>]*> " , string . Empty , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // replace special characters: result = System . Text . RegularExpressions . Regex . Replace ( result , @ " " , " " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " • " , " * " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " ‹ " , " < " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " › " , " > " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " ™ " , " (tm) " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " ⁄ " , " / " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " < " , " < " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " > " , " > " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " © " , " (c) " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , @ " ® " , " (r) " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // Remove all others. More can be added, see // http://hotwired.lycos.com/webmonkey/reference/special_characters/ result = System . Text . RegularExpressions . Regex . Replace ( result , @ " &(.{2,6}); " , string . Empty , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // for testing //System.Text.RegularExpressions.Regex.Replace(result, // this.txtRegex.Text,string.Empty, // System.Text.RegularExpressions.RegexOptions.IgnoreCase); // make line breaking consistent result = result . Replace ( "

" , " \r " ) ; // Remove extra line breaks and tabs: // replace over 2 breaks with 2 and over 4 tabs with 4. // Prepare first to remove any whitespaces in between // the escaped characters and remove redundant tabs in between line breaks result = System . Text . RegularExpressions . Regex . Replace ( result , " (\r)( )+(\r) " , " \r\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , " (\t)( )+(\t) " , " \t\t " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , " (\t)( )+(\r) " , " \t\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; result = System . Text . RegularExpressions . Regex . Replace ( result , " (\r)( )+(\t) " , " \r\t " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // Remove redundant tabs result = System . Text . RegularExpressions . Regex . Replace ( result , " (\r)(\t)+(\r) " , " \r\r " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // Remove multiple tabs following a line break with just one tab result = System . Text . RegularExpressions . Regex . Replace ( result , " (\r)(\t)+ " , " \r\t " , System . Text . RegularExpressions . RegexOptions . IgnoreCase ) ; // Initial replacement target string for line breaks string breaks = " \r\r\r " ; // Initial replacement target string for tabs string tabs = " \t\t\t\t\t " ; for ( int index = 0 ; index < result . Length ; index + + ) { result = result . Replace ( breaks , " \r\r " ) ; result = result . Replace ( tabs , " \t\t\t\t " ) ; breaks = breaks + " \r " ; tabs = tabs + " \t " ; } // That's it. return result ; } catch ( Exception ex ) { log . Error ( ex ) ; return source ; } }