{"id":510502,"date":"2024-07-06T22:45:00","date_gmt":"2024-07-07T05:45:00","guid":{"rendered":"https:\/\/jorgep.com\/blog\/?p=510502"},"modified":"2026-04-15T14:31:03","modified_gmt":"2026-04-15T21:31:03","slug":"how-to-run-llms-on-your-computer","status":"publish","type":"post","link":"https:\/\/jorgep.com\/blog\/how-to-run-llms-on-your-computer\/","title":{"rendered":"How to Run LLMs on Your Computer"},"content":{"rendered":"\n<div class=\"wp-block-columns has-theme-palette-7-background-color has-background is-layout-flex wp-container-core-columns-is-layout-9d6595d7 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<p>Part of: <strong> <a href=\"https:\/\/jorgep.com\/blog\/series-ai-learnings\/\">AI Learning Series Here<\/a><\/strong><\/p>\n\n\n<style>.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col,.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col{flex-direction:column;}.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column395113_43ef2d-d5{position:relative;}@media all and (max-width: 1024px){.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column395113_43ef2d-d5 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column395113_43ef2d-d5\"><div class=\"kt-inside-inner-col\"><style>.wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28, .wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28[data-kb-block=\"kb-adv-heading510545_6813a5-28\"]{font-size:var(--global-kb-font-size-sm, 0.9rem);font-style:normal;}.wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28 mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28[data-kb-block=\"kb-adv-heading510545_6813a5-28\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28 img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading510545_6813a5-28[data-kb-block=\"kb-adv-heading510545_6813a5-28\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading510545_6813a5-28 wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading510545_6813a5-28\">Quick Links:&nbsp;<a href=\"https:\/\/jorgep.com\/blog\/resources-for-learning-ai\/\">Resources for Learning AI<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/keeping-up-with-ai\/\">Keep up with AI<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/list-of-ai-tools\/\" data-type=\"post\" data-id=\"402818\">List of AI Tools<\/a><\/p>\n<\/div><\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\"><div class=\"wp-block-template-part\"><style>.wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47, .wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47[data-kb-block=\"kb-adv-heading395113_c650df-47\"]{text-align:center;font-size:var(--global-kb-font-size-md, 1.25rem);line-height:60px;font-style:normal;background-color:#f5a511;}.wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47 mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47[data-kb-block=\"kb-adv-heading395113_c650df-47\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47 img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading395113_c650df-47[data-kb-block=\"kb-adv-heading395113_c650df-47\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading395113_c650df-47 wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading395113_c650df-47\">Subscribe to <a href=\"https:\/\/go.35s.be\/jtb\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>JorgeTechBits  newsletter<\/strong><\/a><\/p>\n<\/div><\/div>\n<\/div>\n\n\n\n<div style=\"font-family: Verdana, Geneva, sans-serif; font-size: 11px; line-height: 1.6; color: #333;\">\n    <p>\n        <strong>Disclaimer:<\/strong> \n        <em>I personally love to share my learnings, thoughts, and ideas; I get great satisfaction knowing someone has read and benefited from an article. This content is created entirely on my own time and in a personal capacity. The views expressed here are mine alone and do not represent the positions or opinions of my employer.<\/em>\n    <\/p>\n    <p>\n        In my professional role, I serve as a Workforce Transformation Solutions Principal for \n        <a href=\"https:\/\/www.dell.com\/en-us\/work\/learn\/by-service-type-deployment\" style=\"color: #007db8; font-weight: bold; text-decoration: none;\">Dell Technology Services<\/a>. \n        I am passionate about guiding organizations through complex technology transitions and \n        <a href=\"https:\/\/www.delltechnologies.com\/en-us\/what-we-do\/workforce-transformation.htm\" style=\"color: #007db8; font-weight: bold; text-decoration: none;\">Workforce Transformation<\/a>. \n        <a href=\"https:\/\/www.delltechnologies.com\/en-us\/index.htm\" style=\"color: #007db8; font-weight: bold; text-decoration: none;\">Learn more at Dell Technologies<\/a>.\n    <\/p>\n    <hr style=\"border: 0; border-top: 1px solid #ddd; margin: 12px 0;\">\n<\/div>\n\n\n\n<p>Large Language Models (LLMs) have revolutionized the field of natural language processing and artificial intelligence. These powerful models enable applications like language translation, text summarization, and content generation.  The world of Large Language Models (LLMs) can be intimidating, especially with the associated costs. However, there are plenty of free and low-cost options available for those looking to dip their toes into this exciting <strong>field<\/strong>. There are many ways to explore, run and create AI applications on the cloud, there are many benefits of installing LLMs locally on your computer as well&#8230;  <\/p>\n\n\n\n<p>If you want to skip to the list of tools and links below <a href=\"#ToolsList\" data-type=\"internal\" data-id=\"#ToolsList\">click here<\/a><\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Benefits of Installing LLMs Locally<\/h2>\n\n\n<style>.kadence-column510502_afb611-64 > .kt-inside-inner-col,.kadence-column510502_afb611-64 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_afb611-64 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_afb611-64 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_afb611-64 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_afb611-64 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_afb611-64{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_afb611-64 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_afb611-64 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_afb611-64\"><div class=\"kt-inside-inner-col\"><style>.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap{align-content:start;}:where(.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap) > .wp-block-kadence-column{justify-content:start;}.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap{column-gap:var(--global-kb-gap-md, 2rem);row-gap:var(--global-kb-gap-md, 2rem);padding-top:var(--global-kb-spacing-sm, 1.5rem);padding-bottom:var(--global-kb-spacing-sm, 1.5rem);}.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}.kb-row-layout-id510502_265ed3-17 > .kt-row-layout-overlay{opacity:0.30;}@media all and (max-width: 1024px){.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}}@media all and (max-width: 1024px){.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}}@media all and (max-width: 767px){.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_265ed3-17 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr);}}<\/style><div class=\"kb-row-layout-wrap kb-row-layout-id510502_265ed3-17 alignnone wp-block-kadence-rowlayout\"><div class=\"kt-row-column-wrap kt-has-3-columns kt-row-layout-center-exwide kt-tab-layout-inherit kt-mobile-layout-row kt-row-valign-top\">\n<style>.kadence-column510502_433b36-4d > .kt-inside-inner-col,.kadence-column510502_433b36-4d > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_433b36-4d > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_433b36-4d > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_433b36-4d > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_433b36-4d > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_433b36-4d{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_433b36-4d > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_433b36-4d > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_433b36-4d\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n\n<style>.kadence-column510502_c88e66-18 > .kt-inside-inner-col,.kadence-column510502_c88e66-18 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_c88e66-18 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_c88e66-18 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_c88e66-18 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_c88e66-18 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_c88e66-18{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_c88e66-18 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_c88e66-18 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_c88e66-18\"><div class=\"kt-inside-inner-col\">\n<figure class=\"wp-block-table\"><table><thead><tr><th><strong>Category<\/strong><\/th><th><strong>Description<\/strong><\/th><\/tr><\/thead><tbody><tr><td><strong>Privacy &amp; Security<\/strong><\/td><td>Your data never leaves your device<br>Perfect for sensitive personal or business information<br>No need to worry about cloud service privacy policies<\/td><\/tr><tr><td><strong>No Internet Required<\/strong><\/td><td>Work offline without interruption<br>Ideal for travel or areas with poor connectivity<br>Consistent performance regardless of internet speed<\/td><\/tr><tr><td><strong>Cost-Effective<\/strong><\/td><td>No subscription fees or API costs<br>One-time setup with no recurring charges<br>Unlimited usage within your hardware constraints<br><strong>Experimenting without the cost<\/strong><\/td><\/tr><tr><td><strong>Complete Control<\/strong><\/td><td>Customize the model to your specific needs<br>Fine-tune for specialized tasks<br>No content filters or usage restrictions<br>Customizable workflows<br>Experiment with different models<\/td><\/tr><tr><td><strong>Reduced Latency<\/strong><\/td><td>Instant responses without network delays<br>Smoother conversation flow<br>Better integration with local applications<\/td><\/tr><tr><td><\/td><td><\/td><\/tr><tr><td><\/td><td><\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p><\/p>\n<\/div><\/div>\n\n\n<style>.kadence-column510502_cc079a-a6 > .kt-inside-inner-col,.kadence-column510502_cc079a-a6 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_cc079a-a6 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_cc079a-a6 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_cc079a-a6 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_cc079a-a6 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_cc079a-a6{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_cc079a-a6 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_cc079a-a6 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_cc079a-a6\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n<\/div><\/div><\/div><\/div>\n\n\n\n<h2 class=\"wp-block-heading\">Some of the Use Cases to installing and running LLMs Locally:<\/h2>\n\n\n\n<p>Here\u2019s your content formatted into a two-column table:<\/p>\n\n\n<style>.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap{align-content:start;}:where(.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap) > .wp-block-kadence-column{justify-content:start;}.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap{column-gap:var(--global-kb-gap-md, 2rem);row-gap:var(--global-kb-gap-md, 2rem);padding-top:var(--global-kb-spacing-sm, 1.5rem);padding-bottom:var(--global-kb-spacing-sm, 1.5rem);}.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}.kb-row-layout-id510502_f2ae57-4a > .kt-row-layout-overlay{opacity:0.30;}@media all and (max-width: 1024px){.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}}@media all and (max-width: 1024px){.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}}@media all and (max-width: 767px){.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_f2ae57-4a > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr);}}<\/style><div class=\"kb-row-layout-wrap kb-row-layout-id510502_f2ae57-4a alignnone wp-block-kadence-rowlayout\"><div class=\"kt-row-column-wrap kt-has-3-columns kt-row-layout-center-exwide kt-tab-layout-inherit kt-mobile-layout-row kt-row-valign-top\">\n<style>.kadence-column510502_874a09-3e > .kt-inside-inner-col,.kadence-column510502_874a09-3e > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_874a09-3e > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_874a09-3e > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_874a09-3e > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_874a09-3e > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_874a09-3e{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_874a09-3e > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_874a09-3e > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_874a09-3e\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n\n<style>.kadence-column510502_d69c4d-43 > .kt-inside-inner-col,.kadence-column510502_d69c4d-43 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_d69c4d-43 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_d69c4d-43 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_d69c4d-43 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_d69c4d-43 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_d69c4d-43{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_d69c4d-43 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_d69c4d-43 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_d69c4d-43\"><div class=\"kt-inside-inner-col\">\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th class=\"has-text-align-center\" data-align=\"center\"><strong>Personal<\/strong><\/th><th class=\"has-text-align-center\" data-align=\"center\"><strong>Professional<\/strong><\/th><\/tr><\/thead><tbody><tr><td class=\"has-text-align-center\" data-align=\"center\">Writing assistant for offline work<\/td><td class=\"has-text-align-center\" data-align=\"center\">Sensitive document analysis<\/td><\/tr><tr><td class=\"has-text-align-center\" data-align=\"center\">Personal coding companion<\/td><td class=\"has-text-align-center\" data-align=\"center\">Local code review and debugging<\/td><\/tr><tr><td class=\"has-text-align-center\" data-align=\"center\">Local chatbot for learning and study<\/td><td class=\"has-text-align-center\" data-align=\"center\">Customer data processing<\/td><\/tr><tr><td class=\"has-text-align-center\" data-align=\"center\">Personal knowledge base management<\/td><td class=\"has-text-align-center\" data-align=\"center\">Healthcare documentation assistance<\/td><\/tr><tr><td class=\"has-text-align-center\" data-align=\"center\">Creative writing partner<\/td><td class=\"has-text-align-center\" data-align=\"center\">Legal document analysis<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><tbody><tr><td class=\"has-text-align-center\" data-align=\"center\"><strong>Some more generic use cases include:<\/strong><br>Proof of concept development<br>Text summarization: Generate concise summaries of long documents or articles.<br>Language translation: Translate text from one language to another with high accuracy.<br>Content generation: Write articles, blog posts, or even entire books using LLMs as your writing partner.<br>Chatbot development: Utilize LLMs to power conversational AI systems.<br>Data analysis: Process and analyze large datasets with the help of LLMs.<\/td><\/tr><\/tbody><\/table><\/figure>\n<\/div><\/div>\n\n\n<style>.kadence-column510502_d58182-98 > .kt-inside-inner-col,.kadence-column510502_d58182-98 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_d58182-98 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_d58182-98 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_d58182-98 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_d58182-98 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_d58182-98{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_d58182-98 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_d58182-98 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_d58182-98\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n<\/div><\/div>\n\n\n<h3 class=\"wp-block-heading\">How Do Local LLMs Work?<\/h3>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>Model Selection<\/strong>: Choose an LLM that suits your needs. There are various open-source models available, such as GPT-Neo, GPT-J, and LLaMA, that can be downloaded and run locally.<\/li>\n\n\n\n<li><strong>Installation<\/strong>: Set up the necessary software environment. This typically includes installing libraries and frameworks such as Python, PyTorch, TensorFlow, or other specialized tools like Hugging Face Transformers or Ollama, AnytimeLLM and other such tools (see list below) .<\/li>\n\n\n\n<li><strong>Hardware Requirements<\/strong>: Depending on the size of the model and your intended use, you may need a powerful machine with sufficient RAM and a capable GPU. Many models can be resource-intensive, so ensure your hardware can handle the demands.<\/li>\n\n\n\n<li><strong>Loading the Model<\/strong>: Once the environment is set up, load the model into memory. This involves downloading the pre-trained weights and configuration files.<\/li>\n\n\n\n<li><strong>Inference and Fine-Tuning<\/strong>: After loading, you can run inference tasks (like text generation or question answering) directly on your machine. Additionally, many frameworks allow you to fine-tune the model on specific datasets to better fit your needs.<\/li>\n\n\n\n<li><strong>User Interface<\/strong>: For ease of use, some applications provide user interfaces or APIs to interact with the model, making it simpler to integrate into applications or workflows (examples:  AnytimeLLM, LM Studio)<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\">How much hardware do I need to run LLMs locally?<\/h2>\n\n\n\n<p>Tools like Ollama, Anytime LLM, LM Studio make it <strong>SUPER easy to locally run LLMs<\/strong>. These kinds of tools, combined with a technique called <a href=\"https:\/\/jorgep.com\/blog\/what-is-llm-quantization\/\" data-type=\"post\" data-id=\"510503\">LLM Quantization<\/a> (dramatic reduction in model size, decreased memory usage, and improved inference speed)<\/p>\n\n\n\n<p>The hardware requirements are MINIMAL:    I have a small $250 Mini-PC (Celetron N5150 with 16G of RAM) and I can easily run Llama Mistral or Phi3 on it) although it is a bit slow \ud83d\ude42      For most of my learnings I do use a 4-year-old Dell Precision 5540 with 32G RAM and the onboard NVIDIA Quadro GPU<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"ToolsList\">How can I run LLM locally on my machine?<\/h2>\n\n\n\n<p>Here\u2019s a list of applications that allow you to run large language models (LLMs) locally on a Windows (Mac or Linux) device:<\/p>\n\n\n\n<p>PLEASE NOTE that this is just a sampling of what I have used &#8212; There are many others, some specialized.   <br>Update As of 10\/20\/2024:   You can now &#8220;compile&#8221; an LLM and create a chatbot \/ listening stand along application!   See: <strong>llamafile( by Mozilla AI) lets you distribute and run LLMs with a single file. (<a href=\"https:\/\/hacks.mozilla.org\/2023\/11\/introducing-llamafile\/\">announcement blog post<\/a>)<\/strong>  exciting!<\/p>\n\n\n\n<p><br>Some of these applications are available as <a href=\"https:\/\/docs.docker.com\/desktop\/install\/windows-install\/\">Docker containers<\/a>.   Docker Desktop for Windows (Mac, ARM or Linux) is one of the <strong>best ways <\/strong>to try out new applications without affecting or modifying your base OS!  <\/p>\n\n\n<style>.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap{align-content:start;}:where(.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap) > .wp-block-kadence-column{justify-content:start;}.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap{column-gap:var(--global-kb-gap-md, 2rem);row-gap:var(--global-kb-gap-md, 2rem);padding-top:var(--global-kb-spacing-sm, 1.5rem);padding-bottom:var(--global-kb-spacing-sm, 1.5rem);}.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}.kb-row-layout-id510502_931e6c-36 > .kt-row-layout-overlay{opacity:0.30;}@media all and (max-width: 1024px){.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}}@media all and (max-width: 1024px){.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr) minmax(0, 6fr) minmax(0, 1fr);}}@media all and (max-width: 767px){.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap > div:not(.added-for-specificity){grid-column:initial;}.kb-row-layout-id510502_931e6c-36 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr);}}<\/style><div class=\"kb-row-layout-wrap kb-row-layout-id510502_931e6c-36 alignnone wp-block-kadence-rowlayout\"><div class=\"kt-row-column-wrap kt-has-3-columns kt-row-layout-center-exwide kt-tab-layout-inherit kt-mobile-layout-row kt-row-valign-top\">\n<style>.kadence-column510502_21afa2-70 > .kt-inside-inner-col,.kadence-column510502_21afa2-70 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_21afa2-70 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_21afa2-70 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_21afa2-70 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_21afa2-70 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_21afa2-70{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_21afa2-70 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_21afa2-70 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_21afa2-70\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n\n<style>.kadence-column510502_f1424c-ee > .kt-inside-inner-col,.kadence-column510502_f1424c-ee > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_f1424c-ee > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_f1424c-ee > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_f1424c-ee > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_f1424c-ee > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_f1424c-ee{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_f1424c-ee > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_f1424c-ee > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_f1424c-ee\"><div class=\"kt-inside-inner-col\">\n<figure class=\"wp-block-table\"><table class=\"has-theme-palette-7-background-color has-background\"><thead><tr><th><strong>Application<\/strong><\/th><th><strong>Description<\/strong><\/th><\/tr><\/thead><tbody><tr><td><strong><a href=\"https:\/\/ollama.com\/\" data-type=\"link\" data-id=\"https:\/\/ollama.com\/\">Ollama<\/a><\/strong><\/td><td>Command-line (CLI)tool for running LLMs with ease and flexibility. (<a href=\"http:\/\/ApplicationDescriptionOllamaCommand-line (CLI)tool for running LLMs with ease and flexibility. (Installation instructions) hereAnythingLLMAll-in-one AI application that can do RAG, AI Agents, and much more with no code or infrastructure headaches.LM StudioIntegrated environment for experimenting with LLMs locally.Open WebUIWeb-based interface for running various LLMs locally. ( It supports various LLM runners, including Ollama and OpenAI-compatible APIs.)TensorFlowFramework that can be used to train and deploy LLMs locally.\">Installation instructions<\/a>) <\/td><\/tr><tr><td><strong><a href=\"https:\/\/anythingllm.com\/\" data-type=\"link\" data-id=\"https:\/\/anythingllm.com\/\">AnythingLLM<\/a><\/strong><\/td><td>All-in-one AI application that can do RAG, AI Agents, and much more with no code or infrastructure headaches. ( <a href=\"https:\/\/docs.anythingllm.com\/\" data-type=\"link\" data-id=\"https:\/\/docs.anythingllm.com\/\">Docs here<\/a>)<\/td><\/tr><tr><td><strong><a href=\"https:\/\/lmstudio.ai\/\" data-type=\"link\" data-id=\"https:\/\/lmstudio.ai\/\">LM Studio<\/a><\/strong><\/td><td>Integrated environment for experimenting with LLMs locally. (<a href=\"https:\/\/lmstudio.ai\/docs\">Docs here<\/a>)<\/td><\/tr><tr><td><strong><a href=\"https:\/\/docs.openwebui.com\/\">Open WebUI<\/a><\/strong><\/td><td>Web-based interface for running various LLMs locally. It supports various LLM runners, including Ollama and OpenAI-compatible APIs. (<a href=\"https:\/\/docs.openwebui.com\/getting-started\/\">Docs here<\/a>)<\/td><\/tr><tr><td><a href=\"https:\/\/h2o.ai\/platform\/ai-cloud\/make\/llm-studio\/\">H2O LM Studio<\/a><\/td><td>H2O LLM Studio, a framework and no-code GUI designed for<br>fine-tuning state-of-the-art large language models (LLMs). (Ubuntu 16.04 with recent Nvidia drivers.<\/td><\/tr><tr><td><a href=\"https:\/\/www.nomic.ai\/gpt4all\">GPT4ALL<\/a><\/td><td>Framework and chatbot application for all operating systems. We can run the LLMs locally and then use the API to integrate them with any application, such as an AI coding assistant on VSCode.&nbsp;<\/td><\/tr><tr><td><a href=\"https:\/\/jan.ai\/\">Jan.AI<\/a><\/td><td>Jan is an open source ChatGPT-alternative that runs 100% offline.<\/td><\/tr><\/tbody><\/table><\/figure>\n<\/div><\/div>\n\n\n<style>.kadence-column510502_c47429-d0 > .kt-inside-inner-col,.kadence-column510502_c47429-d0 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510502_c47429-d0 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510502_c47429-d0 > .kt-inside-inner-col{flex-direction:column;}.kadence-column510502_c47429-d0 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510502_c47429-d0 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510502_c47429-d0{position:relative;}@media all and (max-width: 1024px){.kadence-column510502_c47429-d0 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510502_c47429-d0 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510502_c47429-d0\"><div class=\"kt-inside-inner-col\"><\/div><\/div>\n\n<\/div><\/div>\n\n\n<p><\/p>\n\n\n\n<h2 class=\"wp-block-heading\">See Also: <\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><a href=\"https:\/\/jorgep.com\/blog\/what-are-large-language-models-llm\/\">What Are Large Language Models (LLM) <\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/jorgep.com\/blog\/chatgpt-is-not-an-llm\/\" data-type=\"post\" data-id=\"464237\">Chatbots are not LLMS<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/jorgep.com\/blog\/create-a-custom-ai-chatbot-using-your-own-data\/\" data-type=\"post\" data-id=\"430244\">Creating a ChatBot using YOUR data<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/jorgep.com\/blog\/copilot-studio-create-your-own-copilot-with-no-code\/\" data-type=\"post\" data-id=\"462663\">Copilot Studio \u2013 Create your own Copilot with No Code <\/a><\/li>\n<\/ul>\n\n\n\n<p>Third party installation guides: (also check YouTube!)<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><a href=\"https:\/\/medium.com\/@researchgraph\/how-to-run-ollama-on-windows-8a1622525ada\">How to run Ollama on Windows. Getting Started with Ollama: A\u2026 | by Research Graph | Medium<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/www.tomsguide.com\/how-to-install-ollama-on-your-computer\">How to install Ollama | Tom&#8217;s Guide<\/a><\/li>\n\n\n\n<li>Video: <a href=\"https:\/\/www.youtube.com\/watch?v=VbfHAHCAYT4&amp;t=93s\">How To Install Any LLM Locally! Open WebUI (Ollama) &#8211; SUPER EASY!<\/a><\/li>\n\n\n\n<li>Video: <a href=\"https:\/\/www.youtube.com\/watch?v=Asleok-Snfs\">Ollama on Windows | Run LLMs locally <\/a><\/li>\n\n\n\n<li>Video: <a href=\"https:\/\/www.youtube.com\/watch?v=zrNKfiCuqCs\">How To Install AI Models with Ollama For Beginners: Get up and running with large language models<\/a><\/li>\n\n\n\n<li>Update 10\/10\/24:  Video: <a href=\"https:\/\/www.youtube.com\/watch?v=mUGsv_IHT-g\">Run Local LLMs on Hardware from $50 to $50,000<\/a><\/li>\n<\/ul>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Large Language Models (LLMs) have revolutionized the field of natural language processing and artificial intelligence. These powerful models enable applications like language translation, text summarization, and content generation. The world of Large Language Models (LLMs) can be intimidating, especially with the associated costs. However, there are plenty of free and low-cost options available for those&#8230;<\/p>\n","protected":false},"author":2,"featured_media":427864,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_kad_blocks_custom_css":"","_kad_blocks_head_custom_js":"","_kad_blocks_body_custom_js":"","_kad_blocks_footer_custom_js":"","ngg_post_thumbnail":0,"_kad_post_transparent":"","_kad_post_title":"","_kad_post_layout":"","_kad_post_sidebar_id":"","_kad_post_content_style":"","_kad_post_vertical_padding":"","_kad_post_feature":"","_kad_post_feature_position":"","_kad_post_header":false,"_kad_post_footer":false,"_kad_post_classname":"","footnotes":""},"categories":[1031,441],"tags":[471,930,842,871,876,986],"class_list":["post-510502","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ai-learnings-series","category-tech-talk","tag-ai","tag-ai-series","tag-chatgpt","tag-genai","tag-llm","tag-local-ai"],"taxonomy_info":{"category":[{"value":1031,"label":"AI Learnings Series"},{"value":441,"label":"Tech Talk"}],"post_tag":[{"value":471,"label":"AI"},{"value":930,"label":"AI Series"},{"value":842,"label":"ChatGPT"},{"value":871,"label":"GenAi"},{"value":876,"label":"LLM"},{"value":986,"label":"Local AI"}]},"featured_image_src_large":["https:\/\/jorgep.com\/blog\/wp-content\/uploads\/FeaturedImage-Topic-AI-1024x512.png",1024,512,true],"author_info":{"display_name":"Jorge Pereira","author_link":"https:\/\/jorgep.com\/blog\/author\/jorge\/"},"comment_info":0,"category_info":[{"term_id":1031,"name":"AI Learnings Series","slug":"ai-learnings-series","term_group":0,"term_taxonomy_id":1041,"taxonomy":"category","description":"","parent":0,"count":9,"filter":"raw","cat_ID":1031,"category_count":9,"category_description":"","cat_name":"AI Learnings Series","category_nicename":"ai-learnings-series","category_parent":0},{"term_id":441,"name":"Tech Talk","slug":"tech-talk","term_group":0,"term_taxonomy_id":451,"taxonomy":"category","description":"","parent":0,"count":684,"filter":"raw","cat_ID":441,"category_count":684,"category_description":"","cat_name":"Tech Talk","category_nicename":"tech-talk","category_parent":0}],"tag_info":[{"term_id":471,"name":"AI","slug":"ai","term_group":0,"term_taxonomy_id":481,"taxonomy":"post_tag","description":"","parent":0,"count":151,"filter":"raw"},{"term_id":930,"name":"AI Series","slug":"ai-series","term_group":0,"term_taxonomy_id":940,"taxonomy":"post_tag","description":"","parent":0,"count":154,"filter":"raw"},{"term_id":842,"name":"ChatGPT","slug":"chatgpt","term_group":0,"term_taxonomy_id":852,"taxonomy":"post_tag","description":"","parent":0,"count":19,"filter":"raw"},{"term_id":871,"name":"GenAi","slug":"genai","term_group":0,"term_taxonomy_id":881,"taxonomy":"post_tag","description":"","parent":0,"count":84,"filter":"raw"},{"term_id":876,"name":"LLM","slug":"llm","term_group":0,"term_taxonomy_id":886,"taxonomy":"post_tag","description":"","parent":0,"count":18,"filter":"raw"},{"term_id":986,"name":"Local AI","slug":"local-ai","term_group":0,"term_taxonomy_id":996,"taxonomy":"post_tag","description":"","parent":0,"count":30,"filter":"raw"}],"_links":{"self":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/510502","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/comments?post=510502"}],"version-history":[{"count":2,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/510502\/revisions"}],"predecessor-version":[{"id":520211,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/510502\/revisions\/520211"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/media\/427864"}],"wp:attachment":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/media?parent=510502"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/categories?post=510502"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/tags?post=510502"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}