Running experiments against e-commerce checkout flows is a common practice, used to improve the efficiency and effectiveness of the checkout process for users. Today, we walk through an example using Winning Variant on Snowflake AI Data Cloud.Watch on YouTube
When developing a new ML model, before putting it out into the wild, it’s important to test its performance against real business metrics. Not just historical testing (what might have happened if we had this version before), but in real time against live activity.Winning Variant allows you to run a split test comparing multiple model versions at the same time against live traffic. The results will provide an objective measure of change of the business metric we care about, and give a more reliable signal we can use to determine if the new model is “better” than the old.Watch on YouTube
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Winning Variant","colors":{"primary":"#0D9373","light":"#07C983","dark":"#0D9373"},"logo":{"light":"https://mintcdn.com/winningvariant/G2QPVHkpV0wa88fg/images/logo-light.svg?fit=max\u0026auto=format\u0026n=G2QPVHkpV0wa88fg\u0026q=85\u0026s=520a80eb34cc0e31441fc0a0b9f956e6","dark":"https://mintcdn.com/winningvariant/G2QPVHkpV0wa88fg/images/logo-dark.svg?fit=max\u0026auto=format\u0026n=G2QPVHkpV0wa88fg\u0026q=85\u0026s=d3b619c36e66e23a6f3c00b72dc7099a"},"favicon":"/favicon.svg","navbar":{"links":[{"href":"https://www.winningvariant.com","label":"Home"}],"primary":{"type":"button","label":"Support","href":"https://www.winningvariant.com/support"}},"footer":{"socials":{"x":"https://x.com/winningvariant","github":"https://github.com/orgs/winningvariant","linkedin":"https://linkedin.com/company/winning-variant"}},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"winningvariant","hasAudioNarration":false,"actualSubdomain":"winningvariant","customDomains":["docs.winningvariant.com"],"gitSource":{"type":"github","owner":"winningvariant","repo":"docs","deployBranch":"main","contentDirectory":"","isPrivate":true,"mountPath":"","label":""},"entitlements":{},"buildId":"6ae922183d1d6aedd3bb90ae:success","clientVersion":"0.0.4102","searchFilterCounts":[{"language":"default","tag":"Documentation","count":19},{"language":"default","tag":"Resources","count":6},{"language":"default","tag":"Analytics","count":5},{"language":"default","tag":"Get Started","count":4},{"language":"default","tag":"Examples","count":4},{"language":"default","tag":"Experimentation Data","count":3},{"language":"default","tag":"Assignments","count":3},{"language":"default","tag":"Linked Identifiers","count":3},{"language":"default","tag":"Advanced","count":2},{"language":"default","tag":"Usage","count":1}]},"assistantConfig":{},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-rbac-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[],"page":{"pagePath":"examples/use-case-walkthrus","slug":"/examples/use-case-walkthrus","pageMetadata":{"title":"Use Case Walkthrus","description":"Full walkthurs of the setup and implementation of various use cases.","href":"/examples/use-case-walkthrus","hidden":false},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Full walkthurs of the setup and implementation of various use cases.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,p:`p`,..._provideComponents(),...props.components},{Heading}=_components;if(!Heading)_missingMdxReference(`Heading`,true);return _jsxs(_Fragment,{children:[_jsx(Heading,{level:`2`,id:`website-checkout-a/b-test-with-apple-pay`,children:`Website Checkout A/B Test with Apple Pay`}),`\n`,_jsx(_components.p,{children:`Running experiments against e-commerce checkout flows is a common practice, used to improve the efficiency and effectiveness of the checkout process for users. Today, we walk through an example using Winning Variant on Snowflake AI Data Cloud.`}),`\n`,_jsx(_components.p,{children:_jsx(_components.a,{href:`https://www.youtube.com/watch?v=1PdVQ2kaUxU`,children:`Watch on YouTube`})}),`\n`,_jsx(Heading,{level:`2`,id:`split-testing-ml-model-versions`,children:`Split Testing ML Model Versions`}),`\n`,_jsx(_components.p,{children:`When developing a new ML model, before putting it out into the wild, it’s important to test its performance against real business metrics. Not just historical testing (what might have happened if we had this version before), but in real time against live activity.`}),`\n`,_jsx(_components.p,{children:`Winning Variant allows you to run a split test comparing multiple model versions at the same time against live traffic. The results will provide an objective measure of change of the business metric we care about, and give a more reliable signal we can use to determine if the new model is “better” than the old.`}),`\n`,_jsx(_components.p,{children:_jsx(_components.a,{href:`https://youtu.be/ttFCFWHTFrI`,children:`Watch on YouTube`})})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Use Case Walkthrus","description":"Full walkthurs of the setup and implementation of various use cases.","href":"/examples/use-case-walkthrus","hidden":false}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Website Checkout A/B Test with Apple Pay","slug":"website-checkout-a/b-test-with-apple-pay","depth":2,"children":[]},{"title":"Split Testing ML Model Versions","slug":"split-testing-ml-model-versions","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":[],"subdomain":"winningvariant","siteName":"Winning Variant","title":"Use Case Walkthrus - Winning Variant","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"originalFileLocation":"examples/use-case-walkthrus.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Full walkthurs of the setup and implementation of various use cases.\">\u003cmeta data-mint-head name=\"application-name\" content=\"Winning Variant\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"Use Case Walkthrus - Winning Variant\">\u003cmeta data-mint-head property=\"og:description\" content=\"Full walkthurs of the setup and implementation of various use cases.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Winning Variant\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.winningvariant.com/examples/use-case-walkthrus\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://winningvariant.mintlify.site/_mintlify/api/og/v2?division=Examples\u0026amp;title=Use+Case+Walkthrus\u0026amp;description=Full+walkthurs+of+the+setup+and+implementation+of+various+use+cases.\u0026amp;theme=b5ae0d55fcc1c5d883047d6b\u0026amp;signature=8a8e732746c0b41d5bd3be0452da2b959354d12d3c85b808a07dd50583e3705f\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Use Case Walkthrus - Winning Variant\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Full walkthurs of the setup and implementation of various use cases.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://winningvariant.mintlify.site/_mintlify/api/og/v2?division=Examples\u0026amp;title=Use+Case+Walkthrus\u0026amp;description=Full+walkthurs+of+the+setup+and+implementation+of+various+use+cases.\u0026amp;theme=b5ae0d55fcc1c5d883047d6b\u0026amp;signature=8a8e732746c0b41d5bd3be0452da2b959354d12d3c85b808a07dd50583e3705f\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/winningvariant/rQ4vFYEZdUJ7y1ks/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Winning Variant\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#0D9373\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.winningvariant.com/examples/use-case-walkthrus\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/examples/use-case-walkthrus.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.winningvariant.com/#organization\",\"name\":\"Winning Variant\",\"url\":\"https://docs.winningvariant.com\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/winningvariant/G2QPVHkpV0wa88fg/images/logo-light.svg?fit=max\u0026auto=format\u0026n=G2QPVHkpV0wa88fg\u0026q=85\u0026s=520a80eb34cc0e31441fc0a0b9f956e6\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.winningvariant.com#website\",\"name\":\"Winning Variant\",\"url\":\"https://docs.winningvariant.com\",\"publisher\":{\"@id\":\"https://docs.winningvariant.com/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.winningvariant.com/examples/use-case-walkthrus#webpage\",\"url\":\"https://docs.winningvariant.com/examples/use-case-walkthrus\",\"name\":\"Use Case Walkthrus\",\"description\":\"Full walkthurs of the setup and implementation of various use cases.\",\"isPartOf\":{\"@id\":\"https://docs.winningvariant.com#website\"},\"breadcrumb\":{\"@id\":\"https://docs.winningvariant.com/examples/use-case-walkthrus#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.winningvariant.com/examples/use-case-walkthrus#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Winning Variant\",\"item\":\"https://docs.winningvariant.com\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Use Case Walkthrus\",\"item\":\"https://docs.winningvariant.com/examples/use-case-walkthrus\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.winningvariant.com/examples/use-case-walkthrus#article\",\"headline\":\"Use Case Walkthrus\",\"name\":\"Use Case Walkthrus\",\"description\":\"Full walkthurs of the setup and implementation of various use cases.\",\"url\":\"https://docs.winningvariant.com/examples/use-case-walkthrus\",\"mainEntityOfPage\":{\"@id\":\"https://docs.winningvariant.com/examples/use-case-walkthrus#webpage\"},\"publisher\":{\"@id\":\"https://docs.winningvariant.com/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.winningvariant.com#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Winning Variant\",\"publisher\":{\"@type\":\"Organization\",\"name\":\"Winning Variant\"},\"provider\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","agentPointerHref":"/examples/use-case-walkthrus.md","siteName":"Winning Variant","nav":{"global":null,"tabs":[{"tab":"Documentation","pages":[{"title":"Documentation","href":"/introduction"}]},{"tab":"Analytics","pages":[{"title":"Analytics","href":"/analytics/dimensions"}]},{"tab":"Examples","groups":[{"group":"Examples","pages":[{"title":"Use Case Walkthrus","href":"/examples/use-case-walkthrus"},{"title":"Split Test AI/ML in Snowflake","description":"Run split tests on Cortex and ML models directly in Snowflake, testing performance against actual business metrics.","href":"/examples/ai-ml"}]}]}]},"basePath":"","title":"Use Case Walkthrus - Winning Variant","buildId":"dpl_596ePT575z4qnAFksq1biRW97aLY","deploymentId":"dpl_596ePT575z4qnAFksq1biRW97aLY","dynamics":[],"chunks":[]}
