feat: adding stream parser (#12666)

Add a stream parser to extract citations (and others) from a stream.
This support cases where markers are split in differen tokens.

Codex never manage to make this code work so everything was done
manually. Please review correctly and do not touch this part of the code
without a very clear understanding of it

codex-rs/utils/stream-parser/src/lib.rs

@@ -0,0 +1,23 @@
+mod assistant_text;
+mod citation;
+mod inline_hidden_tag;
+mod proposed_plan;
+mod stream_text;
+mod tagged_line_parser;
+mod utf8_stream;
+
+pub use assistant_text::AssistantTextChunk;
+pub use assistant_text::AssistantTextStreamParser;
+pub use citation::CitationStreamParser;
+pub use citation::strip_citations;
+pub use inline_hidden_tag::ExtractedInlineTag;
+pub use inline_hidden_tag::InlineHiddenTagParser;
+pub use inline_hidden_tag::InlineTagSpec;
+pub use proposed_plan::ProposedPlanParser;
+pub use proposed_plan::ProposedPlanSegment;
+pub use proposed_plan::extract_proposed_plan_text;
+pub use proposed_plan::strip_proposed_plan_blocks;
+pub use stream_text::StreamTextChunk;
+pub use stream_text::StreamTextParser;
+pub use utf8_stream::Utf8StreamParser;
+pub use utf8_stream::Utf8StreamParserError;