refactor: improve markdown-to-slack conversion by enhancing inline formatting and section handling; add unit test for content order preservation

packages/rerank/src/rerank.test.ts

@@ -10,13 +10,13 @@ const mockCreate = vi.fn();
 describe('Reranker - Unit Tests', () => {
   beforeEach(() => {
     vi.clearAllMocks();
-    
+
     process.env.RERANK_API_KEY = 'test-api-key';
     process.env.RERANK_BASE_URL = 'https://test-api.com/rerank';
     process.env.RERANK_MODEL = 'test-model';
-    
+
     vi.spyOn(console, 'error').mockImplementation(() => {});
-    
+
     const mockAxiosInstance = {
       post: vi.fn(),
     };

packages/slack/src/utils/markdown-to-slack.test.ts

@@ -134,4 +134,87 @@ describe('convertMarkdownToSlack', () => {
 
     expect(result.text).toBe('Text with [link](http://example.com) and ![image](image.png)');
   });
+
+  it('should preserve content order when mixing headers, text, and bullet points', () => {
+    const markdown = `I've created a comprehensive bicycle discontinuation report that identifies which bicycles you should probably stop selling. Here's what the analysis reveals:
+
+## Key Findings
+
+**High-Risk Bicycles for Discontinuation:**
+- **Road-650 series** - Multiple variants showing negative profitability (-16.30% to -1.89%) with extremely low inventory turnover (0.34-0.83)
+- **Touring-3000 series** - Several models with negative profitability (-11.36% to -3.29%) and poor gross margins (-14.15% to -7.69%)
+- **Touring-1000 Yellow variants** - Despite higher sales volume, showing negative profitability (-8.75% to -4.42%)
+
+## Analysis Methodology
+
+I analyzed 97 bicycle products across three categories (Road Bikes, Mountain Bikes, Touring Bikes) using multiple performance metrics:
+
+- **Profitability Index** - Overall financial performance including costs and revenue
+- **Risk Factor** - Warranty and repair costs that impact profitability
+- **Inventory Turnover** - How quickly products sell (below 2.0 indicates slow-moving inventory)
+- **Sales Performance** - Revenue generated over the last 12 months
+- **Gross Margin** - Profit margin percentage
+
+## Discontinuation Recommendations
+
+The report categorizes bicycles into three risk levels:
+
+- **High Risk for Discontinuation** - 25+ bicycles with profitability index below 20, low inventory turnover, or negative margins
+- **Medium Risk** - Products with moderate performance issues
+- **Low Risk** - Well-performing bicycles that should continue
+
+## Dashboard Components
+
+1. **Detailed Analysis Table** - Complete performance metrics for all 97 bicycles with risk categorization
+2. **Top 15 Worst Performers** - Bar chart highlighting bicycles with the lowest profitability
+3. **Category Performance Comparison** - Shows that Mountain Bikes ($5.97M) slightly outperform Touring Bikes ($5.86M) and Road Bikes ($5.52M)
+
+The analysis shows clear patterns: Road-650 and Touring-3000 series consistently underperform across multiple metrics and should be prioritized for discontinuation. Focus on eliminating products with negative profitability indices and inventory turnover below 2.0 to improve overall portfolio performance.`;
+
+    const result = convertMarkdownToSlack(markdown);
+
+    // Should have blocks for headers and content sections
+    expect(result.blocks).toBeDefined();
+    expect(result.blocks!.length).toBeGreaterThan(0);
+
+    // Verify the order is preserved by checking that content sections appear between headers
+    const blockTexts = result
+      .blocks!.filter(
+        (block): block is typeof block & { text: { text: string } } =>
+          'text' in block && block.text !== undefined
+      )
+      .map((block) => block.text.text);
+
+    // First block should be the intro text
+    expect(blockTexts[0]).toContain("I've created a comprehensive bicycle discontinuation report");
+
+    // Then Key Findings header
+    expect(blockTexts[1]).toBe('*Key Findings*');
+
+    // Then the bullet points about High-Risk Bicycles
+    expect(blockTexts[2]).toContain('*High-Risk Bicycles for Discontinuation:*');
+    expect(blockTexts[2]).toContain('• *Road-650 series*');
+    expect(blockTexts[2]).toContain('• *Touring-3000 series*');
+    expect(blockTexts[2]).toContain('• *Touring-1000 Yellow variants*');
+
+    // Then Analysis Methodology header
+    expect(blockTexts[3]).toBe('*Analysis Methodology*');
+
+    // Then the methodology text and bullet points
+    expect(blockTexts[4]).toContain('I analyzed 97 bicycle products');
+    expect(blockTexts[4]).toContain('• *Profitability Index*');
+    expect(blockTexts[4]).toContain('• *Risk Factor*');
+
+    // Verify bullet points are not all grouped at the end
+    const allText = blockTexts.join('\n');
+    const keyFindingsIndex = allText.indexOf('*Key Findings*');
+    const methodologyIndex = allText.indexOf('*Analysis Methodology*');
+    const firstBulletIndex = allText.indexOf('• *Road-650 series*');
+    const methodologyBulletIndex = allText.indexOf('• *Profitability Index*');
+
+    // Bullet points should appear after their respective headers
+    expect(firstBulletIndex).toBeGreaterThan(keyFindingsIndex);
+    expect(firstBulletIndex).toBeLessThan(methodologyIndex);
+    expect(methodologyBulletIndex).toBeGreaterThan(methodologyIndex);
+  });
 });

packages/slack/src/utils/markdown-to-slack.ts

@@ -10,91 +10,141 @@ export function convertMarkdownToSlack(markdown: string): MarkdownConversionResu
     return { text: markdown || '' };
   }
 
-  let text = markdown;
   const blocks: SlackBlock[] = [];
   let hasComplexFormatting = false;
 
-  const headerRegex = /^(#{1,6})\s+(.+)$/gm;
-  const headerMatches = [...text.matchAll(headerRegex)];
+  // Split the content into lines to process sequentially
+  const lines = markdown.split('\n');
+  let currentSection: string[] = [];
 
-  if (headerMatches.length > 0) {
-    hasComplexFormatting = true;
-    for (const match of headerMatches) {
-      const level = match[1]?.length || 1;
-      const headerText = match[2] || '';
+  // Helper function to process inline formatting
+  function processInlineFormatting(text: string): string {
+    let processedText = text;
 
-      const slackHeader = level <= 2 ? `*${headerText}*` : `*${headerText}*`;
+    // Handle code blocks first to prevent interference
+    const codeBlockRegex = /```(\w+)?\n?([\s\S]*?)```/g;
+    processedText = processedText.replace(codeBlockRegex, (_match, language, code) => {
+      return `\`\`\`${language || ''}\n${code.trim()}\n\`\`\``;
+    });
 
+    // Handle inline code
+    const inlineCodeRegex = /`([^`]+)`/g;
+    processedText = processedText.replace(inlineCodeRegex, '`$1`');
+
+    // Handle bold text with placeholder technique
+    const boldPlaceholder = '___BOLD_PLACEHOLDER___';
+    const boldMatches: Array<{ placeholder: string; replacement: string }> = [];
+    let boldCounter = 0;
+
+    const boldRegex = /(\*\*|__)(.*?)\1/g;
+    processedText = processedText.replace(boldRegex, (_match, _delimiter, content) => {
+      const placeholder = `${boldPlaceholder}${boldCounter}${boldPlaceholder}`;
+      const processedContent = content
+        .replace(/\*([^*]+)\*/g, '_$1_')
+        .replace(/_([^_]+)_/g, '_$1_');
+      boldMatches.push({ placeholder, replacement: `*${processedContent}*` });
+      boldCounter++;
+      return placeholder;
+    });
+
+    // Handle italic text
+    const italicRegex = /(?<!\*)\*([^*]+)\*(?!\*)|(?<!_)_([^_]+)_(?!_)/g;
+    processedText = processedText.replace(italicRegex, (_match, group1, group2) => {
+      const content = group1 || group2;
+      return `_${content}_`;
+    });
+
+    // Restore bold text
+    for (const { placeholder, replacement } of boldMatches) {
+      processedText = processedText.replace(placeholder, replacement);
+    }
+
+    return processedText;
+  }
+
+  // Helper function to add current section as a block
+  function flushCurrentSection() {
+    if (currentSection.length > 0) {
+      let sectionText = currentSection.join('\n').trim();
+
+      // Process inline formatting
+      sectionText = processInlineFormatting(sectionText);
+
+      // Process lists
+      const unorderedListRegex = /^[\s]*[-*+]\s+(.+)$/gm;
+      sectionText = sectionText.replace(unorderedListRegex, '• $1');
+
+      const orderedListRegex = /^[\s]*\d+\.\s+(.+)$/gm;
+      let listCounter = 1;
+      sectionText = sectionText.replace(orderedListRegex, (_match, content) => {
+        return `${listCounter++}. ${content}`;
+      });
+
+      if (sectionText) {
+        blocks.push({
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: sectionText,
+          },
+        });
+      }
+      currentSection = [];
+    }
+  }
+
+  // Process lines sequentially
+  for (const line of lines) {
+    // Check if it's a header
+    const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
+    if (headerMatch) {
+      hasComplexFormatting = true;
+
+      // Flush any accumulated content before the header
+      flushCurrentSection();
+
+      // Add the header as its own block
+      const headerText = headerMatch[2] || '';
       blocks.push({
         type: 'section',
         text: {
           type: 'mrkdwn',
-          text: slackHeader,
+          text: `*${headerText}*`,
         },
       });
+    } else {
+      // Accumulate non-header content
+      currentSection.push(line);
     }
-
-    text = text.replace(headerRegex, '');
   }
 
-  const codeBlockRegex = /```(\w+)?\n?([\s\S]*?)```/g;
-  text = text.replace(codeBlockRegex, (_match, language, code) => {
-    return `\`\`\`${language || ''}\n${code.trim()}\n\`\`\``;
-  });
+  // Flush any remaining content
+  flushCurrentSection();
 
-  const inlineCodeRegex = /`([^`]+)`/g;
-  text = text.replace(inlineCodeRegex, '`$1`');
-
-  const boldPlaceholder = '___BOLD_PLACEHOLDER___';
-  const boldMatches: Array<{ placeholder: string; replacement: string }> = [];
-  let boldCounter = 0;
-
-  const boldRegex = /(\*\*|__)(.*?)\1/g;
-  text = text.replace(boldRegex, (_match, _delimiter, content) => {
-    const placeholder = `${boldPlaceholder}${boldCounter}${boldPlaceholder}`;
-    const processedContent = content.replace(/\*([^*]+)\*/g, '_$1_').replace(/_([^_]+)_/g, '_$1_');
-    boldMatches.push({ placeholder, replacement: `*${processedContent}*` });
-    boldCounter++;
-    return placeholder;
-  });
-
-  const italicRegex = /(?<!\*)\*([^*]+)\*(?!\*)|(?<!_)_([^_]+)_(?!_)/g;
-  text = text.replace(italicRegex, (_match, group1, group2) => {
-    const content = group1 || group2;
-    return `_${content}_`;
-  });
-
-  for (const { placeholder, replacement } of boldMatches) {
-    text = text.replace(placeholder, replacement);
-  }
-
-  const unorderedListRegex = /^[\s]*[-*+]\s+(.+)$/gm;
-  text = text.replace(unorderedListRegex, '• $1');
-
-  const orderedListRegex = /^[\s]*\d+\.\s+(.+)$/gm;
-  let listCounter = 1;
-  text = text.replace(orderedListRegex, (_match, content) => {
-    return `${listCounter++}. ${content}`;
-  });
-
-  text = text.replace(/\n{3,}/g, '\n\n').trim();
+  // Clean up any excessive newlines in the final text
+  const fallbackText = markdown.replace(/\n{3,}/g, '\n\n').trim();
 
   if (hasComplexFormatting && blocks.length > 0) {
-    if (text.trim()) {
-      blocks.push({
-        type: 'section',
-        text: {
-          type: 'mrkdwn',
-          text: text.trim(),
-        },
-      });
-    }
-
     return {
-      text: markdown, // Fallback text for notifications
+      text: fallbackText, // Fallback text for notifications
       blocks,
     };
   }
 
-  return { text };
+  // If no complex formatting, process the entire text as one piece
+  let processedText = processInlineFormatting(markdown);
+
+  // Process lists
+  const unorderedListRegex = /^[\s]*[-*+]\s+(.+)$/gm;
+  processedText = processedText.replace(unorderedListRegex, '• $1');
+
+  const orderedListRegex = /^[\s]*\d+\.\s+(.+)$/gm;
+  let listCounter = 1;
+  processedText = processedText.replace(orderedListRegex, (_match, content) => {
+    return `${listCounter++}. ${content}`;
+  });
+
+  processedText = processedText.replace(/\n{3,}/g, '\n\n').trim();
+
+  return { text: processedText };
 }